From 29a092a484eba19f456aedcef5e69298f5f72534 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 9 Jul 2026 23:12:36 +0300 Subject: [PATCH 1/6] Support extra schemas in Ormin imports --- ormin/db_types.nim | 4 +- ormin/db_utils.nim | 4 +- ormin/importer_core.nim | 198 ++++++++++++++++++++++++++++++++++--- ormin/parsesql_tmp.nim | 161 +++++++++++++++++++++++------- tests/tsupabase_import.nim | 87 ++++++++++++++++ 5 files changed, 399 insertions(+), 55 deletions(-) create mode 100644 tests/tsupabase_import.nim diff --git a/ormin/db_types.nim b/ormin/db_types.nim index 29cc10f..df4b5f8 100644 --- a/ormin/db_types.nim +++ b/ormin/db_types.nim @@ -5,7 +5,7 @@ proc dbTypFromName*(name: string): DbTypeKind = var k = dbUnknown case name.toLowerAscii - of "int", "integer", "int8", "smallint", "int16", + of "int", "integer", "int2", "int4", "int8", "smallint", "bigint", "int16", "longint", "int32", "int64", "tinyint", "hugeint": k = dbInt of "uint", "uint8", "uint16", "uint32", "uint64": k = dbUInt of "serial": k = dbSerial @@ -14,7 +14,7 @@ proc dbTypFromName*(name: string): DbTypeKind = of "blob": k = dbBlob of "fixedchar": k = dbFixedChar of "varchar", "text", "string": k = dbVarchar - of "json": k = dbJson + of "json", "jsonb": k = dbJson of "xml": k = dbXml of "decimal": k = dbDecimal of "float", "double", "longdouble", "real": k = dbFloat diff --git a/ormin/db_utils.nim b/ormin/db_utils.nim index 0d0398b..e0fae18 100644 --- a/ormin/db_utils.nim +++ b/ormin/db_utils.nim @@ -113,12 +113,12 @@ iterator tableDefs(sql: DbSql): tuple[name, tableName, model: string] = for i in 0 ..< ast.len: let node = ast[i] if node.kind in {nkCreateTable, nkCreateTableIfNotExists}: - yield (node[0].strVal.toLowerAscii(), $node[0], $node) + yield (sqlIdentBaseName(node[0]).toLowerAscii(), sqlIdentName(node[0]), $node) else: # Fallback: ast might be a single statement (not a list) let node = ast if node.kind in {nkCreateTable, nkCreateTableIfNotExists}: - yield (node[0].strVal.toLowerAscii(), $node[0], $node) + yield (sqlIdentBaseName(node[0]).toLowerAscii(), sqlIdentName(node[0]), $node) iterator tablePairs*(sql: string): tuple[name, model: string] = for name, _, model in tableDefs(DbSql(sql)): diff --git a/ormin/importer_core.nim b/ormin/importer_core.nim index ae91a11..ca36944 100644 --- a/ormin/importer_core.nim +++ b/ormin/importer_core.nim @@ -28,6 +28,7 @@ type DbColumns* = seq[DbColumn] KnownTables* = OrderedTable[string, DbColumns] + KnownEnums = Table[string, seq[string]] ImportTarget* = enum postgre, sqlite, mysql @@ -40,13 +41,172 @@ proc hasRefs(colDesc: SqlNode): (string, string) = for i in 2 ..< colDesc.len: let c = colDesc[i] if c.kind == nkReferences: - if c[0].kind == nkCall: - return (c[0][0].strVal, c[0][1].strVal) - elif c[0].kind == nkIdent: - return ($c[0], "id") + if c[0].kind == nkColumnReference: + return (sqlIdentBaseName(c[0][0]), sqlIdentBaseName(c[0][1])) + elif c[0].kind == nkCall: + return (sqlIdentBaseName(c[0][0]), sqlIdentBaseName(c[0][1])) + elif c[0].kind in {nkIdent, nkQuotedIdent, nkDot}: + return (sqlIdentBaseName(c[0]), "id") ("", "") -proc getType(n: SqlNode): DbType = +proc isSqlIdentChar(c: char): bool = + c in {'a'..'z', 'A'..'Z', '0'..'9', '_', '$'} + +proc skipSqlTrivia(sql: string; pos: var int) = + var keepReading = true + while keepReading and pos < sql.len: + keepReading = false + while pos < sql.len and sql[pos] in Whitespace: + inc pos + keepReading = true + if pos + 1 < sql.len and sql[pos] == '-' and sql[pos + 1] == '-': + inc pos, 2 + while pos < sql.len and sql[pos] notin {'\c', '\L'}: + inc pos + keepReading = true + elif pos + 1 < sql.len and sql[pos] == '/' and sql[pos + 1] == '*': + inc pos, 2 + while pos + 1 < sql.len and not (sql[pos] == '*' and sql[pos + 1] == '/'): + inc pos + if pos + 1 < sql.len: + inc pos, 2 + keepReading = true + +proc consumeSqlKeyword(sql: string; pos: var int; keyword: string): bool = + skipSqlTrivia(sql, pos) + let finish = pos + keyword.len + if finish > sql.len: + return false + if cmpIgnoreCase(sql[pos ..< finish], keyword) != 0: + return false + if finish < sql.len and isSqlIdentChar(sql[finish]): + return false + pos = finish + true + +proc readSqlIdentPart(sql: string; pos: var int): string = + skipSqlTrivia(sql, pos) + if pos >= sql.len: + return "" + + if sql[pos] == '"': + inc pos + while pos < sql.len: + if sql[pos] == '"': + if pos + 1 < sql.len and sql[pos + 1] == '"': + result.add('"') + inc pos, 2 + else: + inc pos + return + else: + result.add(sql[pos]) + inc pos + else: + while pos < sql.len and isSqlIdentChar(sql[pos]): + result.add(sql[pos]) + inc pos + +proc readSqlQualifiedIdent(sql: string; pos: var int): string = + result = readSqlIdentPart(sql, pos) + if result.len == 0: + return + + while true: + let beforeDot = pos + skipSqlTrivia(sql, pos) + if pos >= sql.len or sql[pos] != '.': + pos = beforeDot + return + inc pos + let part = readSqlIdentPart(sql, pos) + if part.len == 0: + return + result.add('.') + result.add(part) + +proc readSqlStringLiteral(sql: string; pos: var int): string = + skipSqlTrivia(sql, pos) + if pos >= sql.len or sql[pos] != '\'': + return "" + + inc pos + while pos < sql.len: + if sql[pos] == '\'': + if pos + 1 < sql.len and sql[pos + 1] == '\'': + result.add('\'') + inc pos, 2 + else: + inc pos + return + else: + result.add(sql[pos]) + inc pos + +proc readSqlEnumValues(sql: string; pos: var int): seq[string] = + skipSqlTrivia(sql, pos) + if pos >= sql.len or sql[pos] != '(': + return @[] + inc pos + + var done = false + while not done and pos < sql.len: + skipSqlTrivia(sql, pos) + if pos < sql.len and sql[pos] == ')': + inc pos + done = true + else: + let value = readSqlStringLiteral(sql, pos) + if value.len == 0: + done = true + else: + result.add(value) + skipSqlTrivia(sql, pos) + if pos < sql.len and sql[pos] == ',': + inc pos + elif pos < sql.len and sql[pos] == ')': + inc pos + done = true + else: + done = true + +proc registerEnum(enums: var KnownEnums; typeName: string; values: seq[string]) = + if typeName.len == 0 or values.len == 0: + return + + let normalized = typeName.toLowerAscii + enums[normalized] = values + let dotPos = normalized.rfind('.') + if dotPos >= 0 and dotPos + 1 < normalized.len: + enums[normalized[dotPos + 1 .. ^1]] = values + +proc collectEnumTypes(schemaSql: string): KnownEnums = + result = initTable[string, seq[string]]() + let lowerSql = schemaSql.toLowerAscii + var searchFrom = 0 + while searchFrom < lowerSql.len: + let found = lowerSql.find("create type", searchFrom) + if found < 0: + break + + var pos = found + if consumeSqlKeyword(schemaSql, pos, "create") and + consumeSqlKeyword(schemaSql, pos, "type"): + let beforeOptional = pos + if consumeSqlKeyword(schemaSql, pos, "if"): + if not consumeSqlKeyword(schemaSql, pos, "not") or + not consumeSqlKeyword(schemaSql, pos, "exists"): + pos = beforeOptional + + let typeName = readSqlQualifiedIdent(schemaSql, pos) + if consumeSqlKeyword(schemaSql, pos, "as") and + consumeSqlKeyword(schemaSql, pos, "enum"): + result.registerEnum(typeName, readSqlEnumValues(schemaSql, pos)) + searchFrom = max(pos, found + 1) + else: + searchFrom = found + 1 + +proc getType(n: SqlNode; enums: KnownEnums): DbType = var it = n if it.kind == nkCall: it = it[0] @@ -56,21 +216,28 @@ proc getType(n: SqlNode): DbType = for i in 0 ..< it.len: assert it[i].kind == nkStringLit result.validValues.add it[i].strVal - elif it.kind in {nkIdent, nkStringLit}: - result.kind = dbTypFromName(it.strVal) - result.name = it.strVal + elif it.kind in {nkIdent, nkQuotedIdent, nkStringLit, nkDot}: + let typeName = sqlIdentName(it) + let normalized = typeName.toLowerAscii + if enums.hasKey(normalized): + result.kind = dbEnum + result.validValues = enums[normalized] + else: + let baseName = sqlIdentBaseName(it) + result.kind = dbTypFromName(baseName) + result.name = typeName -proc collectTables*(n: SqlNode; t: var KnownTables) = +proc collectTables*(n: SqlNode; t: var KnownTables; enums: KnownEnums) = if n.isNil: return case n.kind of nkCreateTable, nkCreateTableIfNotExists: - let tableName = n[0].strVal + let tableName = sqlIdentBaseName(n[0]) var cols: DbColumns = @[] for i in 1 ..< n.len: let it = n[i] if it.kind == nkColumnDef: - var typ = getType(it[1]) + var typ = getType(it[1], enums) if hasAttribute(it, {nkNotNull}): typ.notNull = true cols.add DbColumn( @@ -97,9 +264,9 @@ proc collectTables*(n: SqlNode; t: var KnownTables) = var refTable = "" var refCols: seq[string] = @[] if r.kind == nkColumnReference or r.kind == nkCall: - refTable = r[0].strVal + refTable = sqlIdentBaseName(r[0]) for k in 1 ..< r.len: - refCols.add(r[k].strVal) + refCols.add(sqlIdentBaseName(r[k])) let pairCount = min(localCols.len, refCols.len) for k in 0 ..< pairCount: let localName = localCols[k] @@ -111,7 +278,7 @@ proc collectTables*(n: SqlNode; t: var KnownTables) = t[tableName] = cols else: for i in 0 ..< n.len: - collectTables(n[i], t) + collectTables(n[i], t, enums) proc attrToKey(a: DbColumn; t: KnownTables): int = if a.primaryKey: @@ -128,8 +295,9 @@ proc attrToKey(a: DbColumn; t: KnownTables): int = proc renderModelCode(schemaSql, schemaPath: string; target: ImportTarget; includeStatic = false): string = discard target let sql = parseSql(schemaSql, schemaPath) + let enums = collectEnumTypes(schemaSql) var knownTables = initOrderedTable[string, DbColumns]() - collectTables(sql, knownTables) + collectTables(sql, knownTables, enums) result.add FileHeader result.add "const tableNames = [" diff --git a/ormin/parsesql_tmp.nim b/ormin/parsesql_tmp.nim index b2b4eb9..4c01bf8 100644 --- a/ormin/parsesql_tmp.nim +++ b/ormin/parsesql_tmp.nim @@ -588,6 +588,26 @@ proc `[]`*(n: SqlNode; i: BackwardsIndex): SqlNode = n.sons[n.len - int(i)] proc add*(father, n: SqlNode) = add(father.sons, n) +proc sqlIdentName*(n: SqlNode): string = + ## Return a SQL identifier name from an identifier or dotted identifier node. + case n.kind + of nkIdent, nkQuotedIdent: + result = n.strVal + of nkDot: + result = sqlIdentName(n[0]) & "." & sqlIdentName(n[1]) + else: + result = "" + +proc sqlIdentBaseName*(n: SqlNode): string = + ## Return the unqualified final identifier from an identifier node. + case n.kind + of nkIdent, nkQuotedIdent: + result = n.strVal + of nkDot: + result = sqlIdentBaseName(n[1]) + else: + result = "" + proc getTok(p: var SqlParser) = getTok(p, p.tok) @@ -632,6 +652,49 @@ proc eat(p: var SqlParser, keyw: string) = proc opt(p: var SqlParser, kind: TokKind) = if p.tok.kind == kind: getTok(p) +proc skipToSemicolon(p: var SqlParser) = + while p.tok.kind notin {tkSemicolon, tkEof}: + getTok(p) + +proc skipBalancedParens(p: var SqlParser) = + if p.tok.kind != tkParLe: + return + + var depth = 0 + while p.tok.kind != tkEof: + var shouldReadNext = true + case p.tok.kind + of tkParLe: + inc depth + of tkParRi: + dec depth + getTok(p) + if depth == 0: + return + else: + shouldReadNext = false + else: + discard + if shouldReadNext: + getTok(p) + +proc parseIdentNode(p: var SqlParser): SqlNode = + expectIdent(p) + if p.tok.kind == tkQuotedIdentifier: + result = newNode(nkQuotedIdent, p.tok.literal) + else: + result = newNode(nkIdent, p.tok.literal) + getTok(p) + +proc parseQualifiedIdentifier(p: var SqlParser): SqlNode = + result = parseIdentNode(p) + while p.tok.kind == tkDot: + getTok(p) + let left = result + result = newNode(nkDot) + result.add(left) + result.add(parseIdentNode(p)) + proc parseDataType(p: var SqlParser): SqlNode = if isKeyw(p, "enum"): result = newNode(nkEnumDef) @@ -646,9 +709,7 @@ proc parseDataType(p: var SqlParser): SqlNode = getTok(p) eat(p, tkParRi) else: - expectIdent(p) - result = newNode(nkIdent, p.tok.literal) - getTok(p) + result = parseQualifiedIdentifier(p) if p.tok.kind == tkParLe: var complexType = newNode(nkCall) complexType.add(result) @@ -766,6 +827,10 @@ proc primary(p: var SqlParser): SqlNode = else: sqlError(p, "identifier expected") getTok(p) + of tkColon: + getTok(p) + eat(p, tkColon) + discard parseDataType(p) else: break proc lowestExprAux(p: var SqlParser, v: out SqlNode, limit: int): int = @@ -789,8 +854,7 @@ proc parseExpr(p: var SqlParser): SqlNode = discard lowestExprAux(p, result, - 1) proc parseTableName(p: var SqlParser): SqlNode = - expectIdent(p) - result = primary(p) + result = parseQualifiedIdentifier(p) proc parseColumnReference(p: var SqlParser): SqlNode = result = parseTableName(p) @@ -805,19 +869,38 @@ proc parseColumnReference(p: var SqlParser): SqlNode = result.add(parseTableName(p)) eat(p, tkParRi) +proc parseTableConstraint(p: var SqlParser): SqlNode + proc parseCheck(p: var SqlParser): SqlNode = getTok(p) result = newNode(nkCheck) - result.add(parseExpr(p)) + if p.tok.kind == tkParLe: + skipBalancedParens(p) + result.add(newNode(nkIdent, "true")) + else: + result.add(parseExpr(p)) proc parseConstraint(p: var SqlParser): SqlNode = getTok(p) - result = newNode(nkConstraint) expectIdent(p) - result.add(newNode(nkIdent, p.tok.literal)) + let constraintName = newNode(nkIdent, p.tok.literal) getTok(p) - optKeyw(p, "check") - result.add(parseExpr(p)) + if isKeyw(p, "foreign") or isKeyw(p, "primary") or isKeyw(p, "unique"): + result = parseTableConstraint(p) + elif isKeyw(p, "check"): + result = newNode(nkConstraint) + result.add(constraintName) + let checkNode = parseCheck(p) + if checkNode.len > 0: + result.add(checkNode[0]) + else: + result.add(newNode(nkIdent, "true")) + else: + result = newNode(nkConstraint) + result.add(constraintName) + result.add(newNode(nkIdent, "true")) + while p.tok.kind notin {tkComma, tkParRi, tkEof}: + getTok(p) proc parseParIdentList(p: var SqlParser, father: SqlNode) = eat(p, tkParLe) @@ -937,6 +1020,22 @@ proc parseColumnConstraints(p: var SqlParser, result: SqlNode) = elif isKeyw(p, "identity"): getTok(p) result.add(newNode(nkIdentity)) + elif isKeyw(p, "generated"): + getTok(p) + optKeyw(p, "always") + if isKeyw(p, "by"): + getTok(p) + optKeyw(p, "default") + optKeyw(p, "as") + if isKeyw(p, "identity"): + getTok(p) + result.add(newNode(nkIdentity)) + if p.tok.kind == tkParLe: + skipBalancedParens(p) + elif p.tok.kind == tkParLe: + skipBalancedParens(p) + optKeyw(p, "stored") + optKeyw(p, "virtual") elif isKeyw(p, "primary"): getTok(p) eat(p, "key") @@ -1092,12 +1191,7 @@ proc parseUnique(p: var SqlParser): SqlNode = proc parseTableDef(p: var SqlParser): SqlNode = result = parseIfNotExists(p, nkCreateTable) - expectIdent(p) - if p.tok.kind == tkQuotedIdentifier: - result.add(newNode(nkQuotedIdent, p.tok.literal)) - else: - result.add(newNode(nkIdent, p.tok.literal)) - getTok(p) + result.add(parseQualifiedIdentifier(p)) if p.tok.kind == tkParLe: getTok(p) while p.tok.kind != tkParRi: @@ -1120,9 +1214,7 @@ proc parseTableDef(p: var SqlParser): SqlNode = proc parseTypeDef(p: var SqlParser): SqlNode = result = parseIfNotExists(p, nkCreateType) - expectIdent(p) - result.add(newNode(nkIdent, p.tok.literal)) - getTok(p) + result.add(parseQualifiedIdentifier(p)) eat(p, "as") result.add(parseDataType(p)) @@ -1212,27 +1304,16 @@ proc parseIndexDef(p: var SqlParser): SqlNode = result.add(newNode(nkIdent, p.tok.literal)) getTok(p) eat(p, "on") - expectIdent(p) - result.add(newNode(nkIdent, p.tok.literal)) - getTok(p) + result.add(parseQualifiedIdentifier(p)) eat(p, tkParLe) - expectIdent(p) - result.add(newNode(nkIdent, p.tok.literal)) - getTok(p) - while p.tok.kind == tkComma: - getTok(p) - expectIdent(p) - result.add(newNode(nkIdent, p.tok.literal)) - getTok(p) - eat(p, tkParRi) + skipBalancedParens(p) + skipToSemicolon(p) proc parseInsert(p: var SqlParser): SqlNode = getTok(p) eat(p, "into") - expectIdent(p) result = newNode(nkInsert) - result.add(newNode(nkIdent, p.tok.literal)) - getTok(p) + result.add(parseQualifiedIdentifier(p)) if p.tok.kind == tkParLe: var n = newNode(nkColumnList) parseParIdentList(p, n) @@ -1253,6 +1334,7 @@ proc parseInsert(p: var SqlParser): SqlNode = getTok(p) result.add(n) eat(p, tkParRi) + skipToSemicolon(p) proc parseUpdate(p: var SqlParser): SqlNode = getTok(p) @@ -1401,6 +1483,9 @@ proc parseSelect(p: var SqlParser): SqlNode = proc parseStmt(p: var SqlParser; parent: SqlNode) = if isKeyw(p, "create"): getTok(p) + if isKeyw(p, "or"): + getTok(p) + optKeyw(p, "replace") optKeyw(p, "cached") optKeyw(p, "memory") optKeyw(p, "temp") @@ -1416,7 +1501,7 @@ proc parseStmt(p: var SqlParser; parent: SqlNode) = elif isKeyw(p, "index"): parent.add parseIndexDef(p) else: - sqlError(p, "TABLE expected") + skipToSemicolon(p) elif isKeyw(p, "insert"): parent.add parseInsert(p) elif isKeyw(p, "update"): @@ -1430,8 +1515,12 @@ proc parseStmt(p: var SqlParser; parent: SqlNode) = parent.add parsePragma(p) elif isKeyw(p, "begin"): getTok(p) + elif isKeyw(p, "do") or isKeyw(p, "drop") or isKeyw(p, "alter") or + isKeyw(p, "grant") or isKeyw(p, "revoke") or isKeyw(p, "comment") or + isKeyw(p, "notify") or isKeyw(p, "listen"): + skipToSemicolon(p) else: - sqlError(p, "SELECT, CREATE, UPDATE or DELETE expected") + skipToSemicolon(p) proc parse(p: var SqlParser): SqlNode = ## parses the content of `p`'s input stream and returns the SQL AST. diff --git a/tests/tsupabase_import.nim b/tests/tsupabase_import.nim new file mode 100644 index 0000000..7a00ea7 --- /dev/null +++ b/tests/tsupabase_import.nim @@ -0,0 +1,87 @@ +import std/[assertions, strutils] + +import ormin/importer_core + +const supabaseSchema = """ +do $$ +begin + create type public.service_client_kind as enum ( + 'device', + 'internal_service', + 'partner' + ); +exception + when duplicate_object then null; +end $$; + +do $$ +begin + create type public.service_resource_kind as enum ( + 'platform', + 'organization', + 'device', + 'integration' + ); +exception + when duplicate_object then null; +end $$; + +create table if not exists public.service_clients ( + id uuid primary key default gen_random_uuid(), + client_id text not null unique + check (client_id ~ '^[A-Za-z0-9._:-]{8,128}$'), + kind public.service_client_kind not null, + enabled boolean not null default true, + metadata jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); + +insert into public.service_clients (client_id, kind) +values + ('device-001', 'device'), + ('partner-001', 'partner') +on conflict (client_id) do nothing; + +create table if not exists public.service_client_resource_grants ( + id bigint generated always as identity primary key, + service_client_id uuid not null + references public.service_clients(id) on delete cascade, + resource_kind public.service_resource_kind not null, + organization_id uuid, + device_id text, + integration text, + resource_key text generated always as ( + case resource_kind + when 'platform'::public.service_resource_kind then 'platform' + when 'organization'::public.service_resource_kind then organization_id::text + when 'device'::public.service_resource_kind then organization_id::text || ':' || device_id + when 'integration'::public.service_resource_kind then lower(integration) + end + ) stored, + constraint service_client_resource_grants_shape check ( + resource_kind = 'platform'::public.service_resource_kind + or resource_kind = 'device'::public.service_resource_kind + ) +); + +create index if not exists idx_service_client_resource_grants_device + on public.service_client_resource_grants(organization_id, device_id) + where device_id is not null; + +drop trigger if exists set_service_clients_updated_at on public.service_clients; +create trigger set_service_clients_updated_at +before update on public.service_clients +for each row execute function public.set_updated_at(); +""" + +let schema = supabaseSchema +let model = generateModelCode(schema, "supabase_schema.sql", postgre) + +doAssert model.contains("\"service_clients\"") +doAssert model.contains("\"service_client_resource_grants\"") +doAssert model.contains("Attr(name: \"id\", tabIndex: 0, typ: dbUuid, key: 1)") +doAssert model.contains("Attr(name: \"kind\", tabIndex: 0, typ: dbEnum, key: 0)") +doAssert model.contains("Attr(name: \"metadata\", tabIndex: 0, typ: dbJson, key: 0)") +doAssert model.contains("Attr(name: \"id\", tabIndex: 1, typ: dbInt, key: 1)") +doAssert model.contains("Attr(name: \"resource_kind\", tabIndex: 1, typ: dbEnum, key: 0)") +doAssert model.contains("Attr(name: \"resource_key\", tabIndex: 1, typ: dbVarchar, key: 0)") From 78abd8863473446fd54a286fab273f8cd46d9b36 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 9 Jul 2026 23:33:52 +0300 Subject: [PATCH 2/6] Support extra schemas in Ormin imports --- tests/tsupabase_import.nim | 87 -------------------------------------- 1 file changed, 87 deletions(-) delete mode 100644 tests/tsupabase_import.nim diff --git a/tests/tsupabase_import.nim b/tests/tsupabase_import.nim deleted file mode 100644 index 7a00ea7..0000000 --- a/tests/tsupabase_import.nim +++ /dev/null @@ -1,87 +0,0 @@ -import std/[assertions, strutils] - -import ormin/importer_core - -const supabaseSchema = """ -do $$ -begin - create type public.service_client_kind as enum ( - 'device', - 'internal_service', - 'partner' - ); -exception - when duplicate_object then null; -end $$; - -do $$ -begin - create type public.service_resource_kind as enum ( - 'platform', - 'organization', - 'device', - 'integration' - ); -exception - when duplicate_object then null; -end $$; - -create table if not exists public.service_clients ( - id uuid primary key default gen_random_uuid(), - client_id text not null unique - check (client_id ~ '^[A-Za-z0-9._:-]{8,128}$'), - kind public.service_client_kind not null, - enabled boolean not null default true, - metadata jsonb not null default '{}'::jsonb, - created_at timestamptz not null default now() -); - -insert into public.service_clients (client_id, kind) -values - ('device-001', 'device'), - ('partner-001', 'partner') -on conflict (client_id) do nothing; - -create table if not exists public.service_client_resource_grants ( - id bigint generated always as identity primary key, - service_client_id uuid not null - references public.service_clients(id) on delete cascade, - resource_kind public.service_resource_kind not null, - organization_id uuid, - device_id text, - integration text, - resource_key text generated always as ( - case resource_kind - when 'platform'::public.service_resource_kind then 'platform' - when 'organization'::public.service_resource_kind then organization_id::text - when 'device'::public.service_resource_kind then organization_id::text || ':' || device_id - when 'integration'::public.service_resource_kind then lower(integration) - end - ) stored, - constraint service_client_resource_grants_shape check ( - resource_kind = 'platform'::public.service_resource_kind - or resource_kind = 'device'::public.service_resource_kind - ) -); - -create index if not exists idx_service_client_resource_grants_device - on public.service_client_resource_grants(organization_id, device_id) - where device_id is not null; - -drop trigger if exists set_service_clients_updated_at on public.service_clients; -create trigger set_service_clients_updated_at -before update on public.service_clients -for each row execute function public.set_updated_at(); -""" - -let schema = supabaseSchema -let model = generateModelCode(schema, "supabase_schema.sql", postgre) - -doAssert model.contains("\"service_clients\"") -doAssert model.contains("\"service_client_resource_grants\"") -doAssert model.contains("Attr(name: \"id\", tabIndex: 0, typ: dbUuid, key: 1)") -doAssert model.contains("Attr(name: \"kind\", tabIndex: 0, typ: dbEnum, key: 0)") -doAssert model.contains("Attr(name: \"metadata\", tabIndex: 0, typ: dbJson, key: 0)") -doAssert model.contains("Attr(name: \"id\", tabIndex: 1, typ: dbInt, key: 1)") -doAssert model.contains("Attr(name: \"resource_kind\", tabIndex: 1, typ: dbEnum, key: 0)") -doAssert model.contains("Attr(name: \"resource_key\", tabIndex: 1, typ: dbVarchar, key: 0)") From 2d95072b6727850e67a5c8ac35bab6a313308c1f Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 9 Jul 2026 23:33:58 +0300 Subject: [PATCH 3/6] Support extra schemas in Ormin imports --- tests/tpostgres_schema_import.nim | 87 +++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 tests/tpostgres_schema_import.nim diff --git a/tests/tpostgres_schema_import.nim b/tests/tpostgres_schema_import.nim new file mode 100644 index 0000000..953285b --- /dev/null +++ b/tests/tpostgres_schema_import.nim @@ -0,0 +1,87 @@ +import std/[assertions, strutils] + +import ormin/importer_core + +const postgresSchema = """ +do $$ +begin + create type public.client_kind as enum ( + 'device', + 'internal', + 'partner' + ); +exception + when duplicate_object then null; +end $$; + +do $$ +begin + create type public.resource_kind as enum ( + 'platform', + 'organization', + 'device', + 'integration' + ); +exception + when duplicate_object then null; +end $$; + +create table if not exists public.clients ( + id uuid primary key default gen_random_uuid(), + client_id text not null unique + check (client_id ~ '^[A-Za-z0-9._:-]{8,128}$'), + kind public.client_kind not null, + enabled boolean not null default true, + metadata jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); + +insert into public.clients (client_id, kind) +values + ('device-001', 'device'), + ('partner-001', 'partner') +on conflict (client_id) do nothing; + +create table if not exists public.client_resource_grants ( + id bigint generated always as identity primary key, + service_client_id uuid not null + references public.clients(id) on delete cascade, + resource_kind public.resource_kind not null, + organization_id uuid, + device_id text, + integration text, + resource_key text generated always as ( + case resource_kind + when 'platform'::public.resource_kind then 'platform' + when 'organization'::public.resource_kind then organization_id::text + when 'device'::public.resource_kind then organization_id::text || ':' || device_id + when 'integration'::public.resource_kind then lower(integration) + end + ) stored, + constraint client_resource_grants_shape check ( + resource_kind = 'platform'::public.resource_kind + or resource_kind = 'device'::public.resource_kind + ) +); + +create index if not exists idx_client_resource_grants_device + on public.client_resource_grants(organization_id, device_id) + where device_id is not null; + +drop trigger if exists set_clients_updated_at on public.clients; +create trigger set_clients_updated_at +before update on public.clients +for each row execute function public.set_updated_at(); +""" + +let schema = postgresSchema +let model = generateModelCode(schema, "postgres_schema.sql", postgre) + +doAssert model.contains("\"clients\"") +doAssert model.contains("\"client_resource_grants\"") +doAssert model.contains("Attr(name: \"id\", tabIndex: 0, typ: dbUuid, key: 1)") +doAssert model.contains("Attr(name: \"kind\", tabIndex: 0, typ: dbEnum, key: 0)") +doAssert model.contains("Attr(name: \"metadata\", tabIndex: 0, typ: dbJson, key: 0)") +doAssert model.contains("Attr(name: \"id\", tabIndex: 1, typ: dbInt, key: 1)") +doAssert model.contains("Attr(name: \"resource_kind\", tabIndex: 1, typ: dbEnum, key: 0)") +doAssert model.contains("Attr(name: \"resource_key\", tabIndex: 1, typ: dbVarchar, key: 0)") From f6e324bb57d8cf97894e5c2a8e8573a03f818717 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 9 Jul 2026 23:49:53 +0300 Subject: [PATCH 4/6] Generate database enum values --- ormin/importer_core.nim | 14 ++++++++++++++ ormin/queries.nim | 7 ++++++- tests/tpostgres_schema_import.nim | 23 +++++++++++++++++------ 3 files changed, 37 insertions(+), 7 deletions(-) diff --git a/ormin/importer_core.nim b/ormin/importer_core.nim index ca36944..0df9baf 100644 --- a/ormin/importer_core.nim +++ b/ormin/importer_core.nim @@ -13,6 +13,8 @@ type name: string tabIndex: int typ: DbTypekind + typeName: string + validValues: seq[string] key: int # 0 nothing special, # +1 -- primary key # -N -- references attribute N @@ -292,6 +294,14 @@ proc attrToKey(a: DbColumn; t: KnownTables): int = inc i 0 +proc addStringSeq(dest: var string; values: openArray[string]) = + dest.add "@[" + for i, value in values: + if i > 0: + dest.add ", " + dest.add escape(value) + dest.add "]" + proc renderModelCode(schemaSql, schemaPath: string; target: ImportTarget; includeStatic = false): string = discard target let sql = parseSql(schemaSql, schemaPath) @@ -326,6 +336,10 @@ proc renderModelCode(schemaSql, schemaPath: string; target: ImportTarget; includ result.add $i result.add ", typ: " result.add $a.typ.kind + result.add ", typeName: " + result.add escape(a.typ.name) + result.add ", validValues: " + result.addStringSeq(a.typ.validValues) result.add ", key: " result.add $attrToKey(a, knownTables) result.add ")" diff --git a/ormin/queries.nim b/ormin/queries.nim index 8848126..4f14589 100644 --- a/ormin/queries.nim +++ b/ormin/queries.nim @@ -229,7 +229,12 @@ proc sourceColumns(q: QueryBuilder; source: int): seq[SourceColumn] {.compileTim else: for a in attributes: if a.tabIndex == source: - result.add SourceColumn(name: a.name, typ: DbType(kind: a.typ)) + var typ = DbType(kind: a.typ) + when compiles(a.typeName): + typ.name = a.typeName + when compiles(a.validValues): + typ.validValues = a.validValues + result.add SourceColumn(name: a.name, typ: typ) proc sourceLookup(q: QueryBuilder; table: string): int {.compileTime.} = for i, t in tableNames: diff --git a/tests/tpostgres_schema_import.nim b/tests/tpostgres_schema_import.nim index 953285b..ed940d0 100644 --- a/tests/tpostgres_schema_import.nim +++ b/tests/tpostgres_schema_import.nim @@ -79,9 +79,20 @@ let model = generateModelCode(schema, "postgres_schema.sql", postgre) doAssert model.contains("\"clients\"") doAssert model.contains("\"client_resource_grants\"") -doAssert model.contains("Attr(name: \"id\", tabIndex: 0, typ: dbUuid, key: 1)") -doAssert model.contains("Attr(name: \"kind\", tabIndex: 0, typ: dbEnum, key: 0)") -doAssert model.contains("Attr(name: \"metadata\", tabIndex: 0, typ: dbJson, key: 0)") -doAssert model.contains("Attr(name: \"id\", tabIndex: 1, typ: dbInt, key: 1)") -doAssert model.contains("Attr(name: \"resource_kind\", tabIndex: 1, typ: dbEnum, key: 0)") -doAssert model.contains("Attr(name: \"resource_key\", tabIndex: 1, typ: dbVarchar, key: 0)") +doAssert model.contains("Attr(name: \"id\", tabIndex: 0, typ: dbUuid") +doAssert model.contains("typeName: \"uuid\", validValues: @[], key: 1") +doAssert model.contains("Attr(name: \"kind\", tabIndex: 0, typ: dbEnum") +doAssert model.contains( + "typeName: \"public.client_kind\", validValues: @[" & + "\"device\", \"internal\", \"partner\"]" +) +doAssert model.contains("Attr(name: \"metadata\", tabIndex: 0, typ: dbJson") +doAssert model.contains("typeName: \"jsonb\", validValues: @[], key: 0") +doAssert model.contains("Attr(name: \"id\", tabIndex: 1, typ: dbInt") +doAssert model.contains("typeName: \"bigint\", validValues: @[], key: 1") +doAssert model.contains("Attr(name: \"resource_kind\", tabIndex: 1, typ: dbEnum") +doAssert model.contains( + "typeName: \"public.resource_kind\", validValues: @[" & + "\"platform\", \"organization\", \"device\", \"integration\"]" +) +doAssert model.contains("Attr(name: \"resource_key\", tabIndex: 1, typ: dbVarchar") From 3767fe8170c08723ae243de0a5ee966843f05a80 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Fri, 10 Jul 2026 17:02:56 +0300 Subject: [PATCH 5/6] Fix schema import regressions --- config.nims | 2 + ormin.nimble | 2 +- ormin/importer_core.nim | 138 +++++++++++++++++++++------- ormin/parsesql_tmp.nim | 106 ++++++++++++++++++++- ormin/queries.nim | 46 ++++++++-- tests/qualified_schema_model.sql | 11 +++ tests/tdb_utils.nim | 18 +++- tests/tpostgres_schema_import.nim | 40 +++++++- tests/tqualified_schema_queries.nim | 31 +++++++ 9 files changed, 343 insertions(+), 51 deletions(-) create mode 100644 tests/qualified_schema_model.sql create mode 100644 tests/tqualified_schema_queries.nim diff --git a/config.nims b/config.nims index 9eb46ad..0377d94 100644 --- a/config.nims +++ b/config.nims @@ -20,6 +20,8 @@ task test, "Run all test suite": exec "nim c -f -r tests/tsqlite" exec "nim c -f -r tests/tdb_utils" exec "nim c -f -r tests/timportstatic" + exec "nim c -f -r tests/tpostgres_schema_import" + exec "nim c -f -r tests/tqualified_schema_queries" task setup_postgres, "Ensure local Postgres has test DB/user": # Use a simple script to avoid Nim/psql quoting pitfalls diff --git a/ormin.nimble b/ormin.nimble index 1de2b6e..dcb6a15 100644 --- a/ormin.nimble +++ b/ormin.nimble @@ -1,6 +1,6 @@ # Package -version = "0.9.0" +version = "0.10.0" author = "Araq" description = "Prepared SQL statement generator. A lightweight ORM." license = "MIT" diff --git a/ormin/importer_core.nim b/ormin/importer_core.nim index 0df9baf..558fc78 100644 --- a/ormin/importer_core.nim +++ b/ormin/importer_core.nim @@ -44,11 +44,11 @@ proc hasRefs(colDesc: SqlNode): (string, string) = let c = colDesc[i] if c.kind == nkReferences: if c[0].kind == nkColumnReference: - return (sqlIdentBaseName(c[0][0]), sqlIdentBaseName(c[0][1])) + return (sqlIdentName(c[0][0]), sqlIdentBaseName(c[0][1])) elif c[0].kind == nkCall: - return (sqlIdentBaseName(c[0][0]), sqlIdentBaseName(c[0][1])) + return (sqlIdentName(c[0][0]), sqlIdentBaseName(c[0][1])) elif c[0].kind in {nkIdent, nkQuotedIdent, nkDot}: - return (sqlIdentBaseName(c[0]), "id") + return (sqlIdentName(c[0]), "id") ("", "") proc isSqlIdentChar(c: char): bool = @@ -127,24 +127,47 @@ proc readSqlQualifiedIdent(sql: string; pos: var int): string = result.add('.') result.add(part) -proc readSqlStringLiteral(sql: string; pos: var int): string = +proc readSqlStringLiteral(sql: string; pos: var int; value: var string): bool = skipSqlTrivia(sql, pos) if pos >= sql.len or sql[pos] != '\'': - return "" + return false inc pos while pos < sql.len: if sql[pos] == '\'': if pos + 1 < sql.len and sql[pos + 1] == '\'': - result.add('\'') + value.add('\'') inc pos, 2 else: inc pos - return + return true else: - result.add(sql[pos]) + value.add(sql[pos]) inc pos +proc readDollarQuotedBody(sql: string; pos: var int; body: var string): bool = + if pos >= sql.len or sql[pos] != '$': + return false + + let delimiterStart = pos + inc pos + while pos < sql.len and sql[pos] in {'a'..'z', 'A'..'Z', '0'..'9', '_'}: + inc pos + if pos >= sql.len or sql[pos] != '$': + pos = delimiterStart + return false + + inc pos + let delimiter = sql[delimiterStart ..< pos] + let bodyEnd = sql.find(delimiter, pos) + if bodyEnd < 0: + pos = delimiterStart + return false + + body = sql[pos ..< bodyEnd] + pos = bodyEnd + delimiter.len + true + proc readSqlEnumValues(sql: string; pos: var int): seq[string] = skipSqlTrivia(sql, pos) if pos >= sql.len or sql[pos] != '(': @@ -158,8 +181,8 @@ proc readSqlEnumValues(sql: string; pos: var int): seq[string] = inc pos done = true else: - let value = readSqlStringLiteral(sql, pos) - if value.len == 0: + var value = "" + if not readSqlStringLiteral(sql, pos, value): done = true else: result.add(value) @@ -182,31 +205,56 @@ proc registerEnum(enums: var KnownEnums; typeName: string; values: seq[string]) if dotPos >= 0 and dotPos + 1 < normalized.len: enums[normalized[dotPos + 1 .. ^1]] = values -proc collectEnumTypes(schemaSql: string): KnownEnums = - result = initTable[string, seq[string]]() - let lowerSql = schemaSql.toLowerAscii - var searchFrom = 0 - while searchFrom < lowerSql.len: - let found = lowerSql.find("create type", searchFrom) - if found < 0: +proc collectEnumTypes(schemaSql: string; enums: var KnownEnums) = + var pos = 0 + while pos < schemaSql.len: + skipSqlTrivia(schemaSql, pos) + if pos >= schemaSql.len: break - var pos = found - if consumeSqlKeyword(schemaSql, pos, "create") and - consumeSqlKeyword(schemaSql, pos, "type"): - let beforeOptional = pos - if consumeSqlKeyword(schemaSql, pos, "if"): - if not consumeSqlKeyword(schemaSql, pos, "not") or - not consumeSqlKeyword(schemaSql, pos, "exists"): - pos = beforeOptional - - let typeName = readSqlQualifiedIdent(schemaSql, pos) - if consumeSqlKeyword(schemaSql, pos, "as") and - consumeSqlKeyword(schemaSql, pos, "enum"): - result.registerEnum(typeName, readSqlEnumValues(schemaSql, pos)) - searchFrom = max(pos, found + 1) + case schemaSql[pos] + of '\'': + var ignored = "" + if not readSqlStringLiteral(schemaSql, pos, ignored): + inc pos + of '"': + discard readSqlIdentPart(schemaSql, pos) + of '$': + var body = "" + if readDollarQuotedBody(schemaSql, pos, body): + collectEnumTypes(body, enums) + else: + inc pos else: - searchFrom = found + 1 + if schemaSql[pos] notin {'a'..'z', 'A'..'Z', '_'}: + inc pos + continue + + let wordStart = pos + while pos < schemaSql.len and isSqlIdentChar(schemaSql[pos]): + inc pos + if cmpIgnoreCase(schemaSql[wordStart ..< pos], "create") != 0: + continue + + var declarationPos = pos + if not consumeSqlKeyword(schemaSql, declarationPos, "type"): + continue + + let beforeOptional = declarationPos + if consumeSqlKeyword(schemaSql, declarationPos, "if"): + if not consumeSqlKeyword(schemaSql, declarationPos, "not") or + not consumeSqlKeyword(schemaSql, declarationPos, "exists"): + declarationPos = beforeOptional + + let typeName = readSqlQualifiedIdent(schemaSql, declarationPos) + if consumeSqlKeyword(schemaSql, declarationPos, "as") and + consumeSqlKeyword(schemaSql, declarationPos, "enum"): + enums.registerEnum(typeName, readSqlEnumValues(schemaSql, declarationPos)) + pos = max(pos, declarationPos) + +proc collectEnumTypes(schemaSql: string): KnownEnums = + result = initTable[string, seq[string]]() + collectEnumTypes(schemaSql, result) proc getType(n: SqlNode; enums: KnownEnums): DbType = var it = n @@ -234,7 +282,7 @@ proc collectTables*(n: SqlNode; t: var KnownTables; enums: KnownEnums) = return case n.kind of nkCreateTable, nkCreateTableIfNotExists: - let tableName = sqlIdentBaseName(n[0]) + let tableName = sqlIdentName(n[0]) var cols: DbColumns = @[] for i in 1 ..< n.len: let it = n[i] @@ -266,7 +314,7 @@ proc collectTables*(n: SqlNode; t: var KnownTables; enums: KnownEnums) = var refTable = "" var refCols: seq[string] = @[] if r.kind == nkColumnReference or r.kind == nkCall: - refTable = sqlIdentBaseName(r[0]) + refTable = sqlIdentName(r[0]) for k in 1 ..< r.len: refCols.add(sqlIdentBaseName(r[k])) let pairCount = min(localCols.len, refCols.len) @@ -286,10 +334,30 @@ proc attrToKey(a: DbColumn; t: KnownTables): int = if a.primaryKey: return 1 if a.refs[0].len > 0: + var referencedTable = "" + for tableName in keys(t): + if cmpIgnoreCase(tableName, a.refs[0]) == 0: + referencedTable = tableName + break + + if referencedTable.len == 0 and '.' notin a.refs[0]: + for tableName in keys(t): + let dotPos = tableName.rfind('.') + let baseName = + if dotPos >= 0: tableName[dotPos + 1 .. ^1] + else: tableName + if cmpIgnoreCase(baseName, a.refs[0]) == 0: + if referencedTable.len > 0: + return 0 + referencedTable = tableName + + if referencedTable.len == 0: + return 0 + var i = 0 for k, v in pairs(t): for b in v: - if cmpIgnoreCase(k, a.refs[0]) == 0 and cmpIgnoreCase(b.name, a.refs[1]) == 0: + if cmpIgnoreCase(k, referencedTable) == 0 and cmpIgnoreCase(b.name, a.refs[1]) == 0: return -i - 1 inc i 0 diff --git a/ormin/parsesql_tmp.nim b/ormin/parsesql_tmp.nim index 4c01bf8..d304fe9 100644 --- a/ormin/parsesql_tmp.nim +++ b/ormin/parsesql_tmp.nim @@ -477,6 +477,7 @@ type nkHexStringLit, nkIntegerLit, nkNumericLit, + nkRaw, nkPrimaryKey, nkForeignKey, nkNotNull, @@ -540,7 +541,7 @@ type const LiteralNodes = { nkIdent, nkQuotedIdent, nkStringLit, nkBitStringLit, nkHexStringLit, - nkIntegerLit, nkNumericLit + nkIntegerLit, nkNumericLit, nkRaw } type @@ -678,6 +679,102 @@ proc skipBalancedParens(p: var SqlParser) = if shouldReadNext: getTok(p) +proc stripTrailingSqlSpace(sql: var string) = + while sql.len > 0 and sql[^1] in Whitespace: + sql.setLen(sql.len - 1) + +proc addSqlStringLiteral(sql: var string; value: string; prefix = "") = + sql.add(prefix) + sql.add('\'') + sql.add(value.replace("'", "''")) + sql.add('\'') + +proc addSqlToken(sql: var string; tok: Token) = + case tok.kind + of tkParLe: + sql.add('(') + of tkParRi: + sql.stripTrailingSqlSpace() + sql.add(')') + of tkBracketLe: + sql.add('[') + of tkBracketRi: + sql.stripTrailingSqlSpace() + sql.add(']') + of tkComma: + sql.stripTrailingSqlSpace() + sql.add(", ") + of tkDot, tkColon: + sql.stripTrailingSqlSpace() + sql.add(tok.literal) + of tkOperator: + if sql.len > 0 and sql[^1] notin Whitespace + {'(', '[', '.', ':'}: + sql.add(' ') + sql.add(tok.literal) + sql.add(' ') + of tkQuotedIdentifier: + if sql.len > 0 and sql[^1] notin Whitespace + {'(', '[', '.', ':'}: + sql.add(' ') + sql.add('"') + sql.add(tok.literal.replace("\"", "\"\"")) + sql.add('"') + of tkStringConstant: + if sql.len > 0 and sql[^1] notin Whitespace + {'(', '[', '.', ':'}: + sql.add(' ') + sql.addSqlStringLiteral(tok.literal) + of tkEscapeConstant: + if sql.len > 0 and sql[^1] notin Whitespace + {'(', '[', '.', ':'}: + sql.add(' ') + var escaped = tok.literal.replace("\\", "\\\\") + escaped = escaped.replace("'", "''") + sql.add("E'") + sql.add(escaped) + sql.add('\'') + of tkDollarQuotedConstant: + if sql.len > 0 and sql[^1] notin Whitespace + {'(', '[', '.', ':'}: + sql.add(' ') + var tag = "$ormin$" + while tag in tok.literal: + tag = tag[0 ..< ^1] & "_or$" + sql.add(tag) + sql.add(tok.literal) + sql.add(tag) + of tkBitStringConstant: + if sql.len > 0 and sql[^1] notin Whitespace + {'(', '[', '.', ':'}: + sql.add(' ') + sql.addSqlStringLiteral(tok.literal, "B") + of tkHexStringConstant: + if sql.len > 0 and sql[^1] notin Whitespace + {'(', '[', '.', ':'}: + sql.add(' ') + sql.addSqlStringLiteral(tok.literal, "X") + of tkEof: + discard + else: + if sql.len > 0 and sql[^1] notin Whitespace + {'(', '[', '.', ':'}: + sql.add(' ') + sql.add(tok.literal) + +proc readBalancedSql(p: var SqlParser): string = + if p.tok.kind != tkParLe: + return + + var depth = 0 + while p.tok.kind != tkEof: + let kind = p.tok.kind + case kind + of tkParLe: + inc depth + of tkParRi: + dec depth + else: + discard + result.addSqlToken(p.tok) + getTok(p) + if kind == tkParRi and depth == 0: + return + + sqlError(p, "closing parenthesis expected") + proc parseIdentNode(p: var SqlParser): SqlNode = expectIdent(p) if p.tok.kind == tkQuotedIdentifier: @@ -875,8 +972,7 @@ proc parseCheck(p: var SqlParser): SqlNode = getTok(p) result = newNode(nkCheck) if p.tok.kind == tkParLe: - skipBalancedParens(p) - result.add(newNode(nkIdent, "true")) + result.add(newNode(nkRaw, readBalancedSql(p))) else: result.add(parseExpr(p)) @@ -1175,7 +1271,7 @@ proc parseTableConstraint(p: var SqlParser): SqlNode = result.add(m) elif isKeyw(p, "unique"): getTok(p) - eat(p, "key") + optKeyw(p, "key") result = newNode(nkUnique) parseParIdentList(p, result) elif isKeyw(p, "check"): @@ -1623,6 +1719,8 @@ proc ra(n: SqlNode, s: var SqlWriter) = s.add("x'" & n.strVal & "'") of nkIntegerLit, nkNumericLit: s.add(n.strVal) + of nkRaw: + s.add(n.strVal) of nkPrimaryKey: s.addKeyw("primary key") rs(n, s) diff --git a/ormin/queries.nim b/ormin/queries.nim index 4f14589..85a7fc6 100644 --- a/ormin/queries.nim +++ b/ormin/queries.nim @@ -217,6 +217,13 @@ proc lookupCte(ctes: openArray[CteDef]; name: string): int {.compileTime.} = if cmpIgnoreCase(cte.name, name) == 0: return i +proc baseTableName(name: string): string {.compileTime.} = + let dotPos = name.rfind('.') + if dotPos >= 0: + result = name[dotPos + 1 .. ^1] + else: + result = name + proc sourceName(q: QueryBuilder; source: int): string {.compileTime.} = if isCteEnvIndex(source): result = q.ctes[fromCteEnvIndex(source)].name @@ -240,11 +247,28 @@ proc sourceLookup(q: QueryBuilder; table: string): int {.compileTime.} = for i, t in tableNames: if cmpIgnoreCase(t, table) == 0: return i + let cteIdx = lookupCte(q.ctes, table) if cteIdx >= 0: return cteEnvIndex(cteIdx) + + var baseMatch = -1 + if '.' notin table: + for i, t in tableNames: + if cmpIgnoreCase(baseTableName(t), table) == 0: + if baseMatch >= 0: + return -1 + baseMatch = i + if baseMatch >= 0: + return baseMatch result = -1 +proc sourceMatches(q: QueryBuilder; source: int; table: string): bool {.compileTime.} = + let name = sourceName(q, source) + result = cmpIgnoreCase(name, table) == 0 + if not result and not isCteEnvIndex(source) and '.' notin table: + result = cmpIgnoreCase(baseTableName(name), table) == 0 + proc sourceAlias(q: QueryBuilder; source: int; sourceName: string): string {.compileTime.} = if q.kind == qkJoin and q.env.len > 0 and q.env[^1][0] == source: result = q.env[^1][1] @@ -275,7 +299,7 @@ proc lookup(table, attr: string; qb: QueryBuilder; alias: var string): DbType = var found = false var foundSource = -1 for e in qb.env: - if table.len == 0 or cmpIgnoreCase(sourceName(qb, e[0]), table) == 0: + if table.len == 0 or sourceMatches(qb, e[0], table): for col in sourceColumns(qb, e[0]): if cmpIgnoreCase(col.name, attr) == 0: if found: @@ -364,6 +388,12 @@ proc nodeName(n: NimNode): string {.compileTime.} = result = nodeName(n[0]) else: result = "" + of nnkDotExpr: + if n.len == 2: + let left = nodeName(n[0]) + let right = nodeName(n[1]) + if left.len > 0 and right.len > 0: + result = left & "." & right else: result = "" @@ -511,8 +541,8 @@ proc cond(n: NimNode; q: var string; params: var Params; else: result = lookupColumnInEnv(n, q, params, expected, qb) of nnkDotExpr: - let t = $n[0] - let a = $n[1] + let t = nodeName(n[0]) + let a = nodeName(n[1]) escIdent(q, t) q.add '.' escIdent(q, a) @@ -927,7 +957,7 @@ proc selectAll(q: QueryBuilder; tabIndex: int; arg, lineInfo: NimNode) = proc tableSel(n: NimNode; q: QueryBuilder) = if n.kind == nnkCall and q.kind != qkDelete: let call = n - let tab = $call[0] + let tab = nodeName(call[0]) let tabIndex = sourceLookup(q, tab) if tabIndex < 0: macros.error "unknown table name: " & tab & " from: " & fmtTableList(tableNames), n @@ -1014,8 +1044,8 @@ proc tableSel(n: NimNode; q: QueryBuilder) = else: macros.error "unknown selector: " & repr(n), n if q.kind notin {qkUpdate, qkSelect, qkJoin}: q.head.add ")" - elif n.kind in {nnkIdent, nnkAccQuoted, nnkSym} and q.kind == qkDelete: - let tab = $n + elif n.kind in {nnkIdent, nnkAccQuoted, nnkSym, nnkDotExpr} and q.kind == qkDelete: + let tab = nodeName(n) let tabIndex = sourceLookup(q, tab) if tabIndex < 0: macros.error "unknown table name: " & tab & " from: " & fmtTableList(tableNames), n @@ -1142,7 +1172,7 @@ proc queryh(n: NimNode; q: QueryBuilder) = if joinClause.kind == nnkCommand and joinClause.len == 2 and joinClause[1].kind == nnkCommand and joinClause[1].len == 2 and $joinClause[1][0] == "on" and joinClause[0].kind == nnkCall: - let tab = $joinClause[0][0] + let tab = nodeName(joinClause[0][0]) let tabIndex = sourceLookup(q, tab) if tabIndex < 0: macros.error "unknown table name: " & tab & " from: " & fmtTableList(tableNames), n @@ -1163,7 +1193,7 @@ proc queryh(n: NimNode; q: QueryBuilder) = swap q.env, oldEnv checkBool(t, onn) elif joinClause.kind == nnkCall: - let tab = $joinClause[0] + let tab = nodeName(joinClause[0]) let tabIndex = sourceLookup(q, tab) if tabIndex < 0: macros.error "unknown table name: " & tab & " from: " & fmtTableList(tableNames), n[1][0] diff --git a/tests/qualified_schema_model.sql b/tests/qualified_schema_model.sql new file mode 100644 index 0000000..ef90b85 --- /dev/null +++ b/tests/qualified_schema_model.sql @@ -0,0 +1,11 @@ +create table public.events ( + public_value text +); + +create table audit.events ( + audit_value text +); + +create table public.users ( + username text +); diff --git a/tests/tdb_utils.nim b/tests/tdb_utils.nim index 525c1ef..45c037f 100644 --- a/tests/tdb_utils.nim +++ b/tests/tdb_utils.nim @@ -1,4 +1,4 @@ -import unittest, os, sequtils +import std/[assertions, os, sequtils, strutils, unittest] import db_connector/db_common from db_connector/db_sqlite import open, exec, getValue import ormin/db_utils @@ -41,6 +41,22 @@ let sqlContent = """ const staticSqlContent = staticLoad("db_utils_case_quoted.sql") +block checkConstraintRoundTrip: + const schema = """ +create table accounts ( + balance integer check (balance >= 0), + code text constraint normalized_code check ( + code ~ '^[A-Z]+$' and code = 'X'::text + ) +); +""" + let pairs = tablePairs(schema).toSeq() + doAssert pairs.len == 1 + doAssert pairs[0].model.contains("balance >= 0") + doAssert pairs[0].model.contains("code ~ '^[A-Z]+$'") + doAssert pairs[0].model.contains("'X'::text") + doAssert not pairs[0].model.contains("check true") + writeFile($sqlFile, sqlContent) suite "db_utils: case and quoted names": diff --git a/tests/tpostgres_schema_import.nim b/tests/tpostgres_schema_import.nim index ed940d0..569e23e 100644 --- a/tests/tpostgres_schema_import.nim +++ b/tests/tpostgres_schema_import.nim @@ -77,8 +77,8 @@ for each row execute function public.set_updated_at(); let schema = postgresSchema let model = generateModelCode(schema, "postgres_schema.sql", postgre) -doAssert model.contains("\"clients\"") -doAssert model.contains("\"client_resource_grants\"") +doAssert model.contains("\"public.clients\"") +doAssert model.contains("\"public.client_resource_grants\"") doAssert model.contains("Attr(name: \"id\", tabIndex: 0, typ: dbUuid") doAssert model.contains("typeName: \"uuid\", validValues: @[], key: 1") doAssert model.contains("Attr(name: \"kind\", tabIndex: 0, typ: dbEnum") @@ -96,3 +96,39 @@ doAssert model.contains( "\"platform\", \"organization\", \"device\", \"integration\"]" ) doAssert model.contains("Attr(name: \"resource_key\", tabIndex: 1, typ: dbVarchar") + +block qualifiedTableNamesRemainDistinct: + const schemaText = """ +create table public.events (public_value text); +create table audit.events (audit_value text); +""" + let schema = schemaText + let generated = generateModelCode(schema, "qualified.sql", postgre) + doAssert generated.contains("\"public.events\"") + doAssert generated.contains("\"audit.events\"") + doAssert generated.contains("Attr(name: \"public_value\", tabIndex: 0") + doAssert generated.contains("Attr(name: \"audit_value\", tabIndex: 1") + +block enumKeywordsMayBeSeparatedByWhitespace: + const schemaText = """ +create +type public.mood as enum ('happy', 'sad'); +create table public.people (mood public.mood); +""" + let schema = schemaText + let generated = generateModelCode(schema, "enum_whitespace.sql", postgre) + doAssert generated.contains("Attr(name: \"mood\", tabIndex: 0, typ: dbEnum") + doAssert generated.contains( + "typeName: \"public.mood\", validValues: @[\"happy\", \"sad\"]" + ) + +block namedUniqueConstraintImports: + const schemaText = """ +create table public.people ( + email text, + constraint people_email_key unique (email) +); +""" + let schema = schemaText + let generated = generateModelCode(schema, "named_unique.sql", postgre) + doAssert generated.contains("Attr(name: \"email\", tabIndex: 0, typ: dbVarchar") diff --git a/tests/tqualified_schema_queries.nim b/tests/tqualified_schema_queries.nim new file mode 100644 index 0000000..663a18a --- /dev/null +++ b/tests/tqualified_schema_queries.nim @@ -0,0 +1,31 @@ +import std/assertions + +import ormin + +importModel(DbBackend.postgre, "qualified_schema_model", includeStatic = true) + +var db {.global.}: DbConn + +proc selectPublicEvents() = + discard query: + select public.events(public_value) + where public.events.public_value == "visible" + +proc selectAuditEvents() = + discard query: + select audit.events(audit_value) + +proc selectUsersByUnambiguousBaseName() = + discard query: + select users(username) + +proc joinQualifiedEvents() = + discard query: + select public.events(public_value) + join audit.events(audit_value) on public.events.public_value == audit.events.audit_value + +static: + doAssert not compiles(block: + discard query: + select events(public_value) + ) From 462bece47f66307bea7e94c9d5a2d273ad71c235 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Fri, 10 Jul 2026 18:49:42 +0300 Subject: [PATCH 6/6] Reuse SQL lexer for enum discovery --- ormin/importer_core.nim | 214 +++------------------------------------- ormin/parsesql_tmp.nim | 29 ++++++ 2 files changed, 41 insertions(+), 202 deletions(-) diff --git a/ormin/importer_core.nim b/ormin/importer_core.nim index 558fc78..901349a 100644 --- a/ormin/importer_core.nim +++ b/ormin/importer_core.nim @@ -51,210 +51,20 @@ proc hasRefs(colDesc: SqlNode): (string, string) = return (sqlIdentName(c[0]), "id") ("", "") -proc isSqlIdentChar(c: char): bool = - c in {'a'..'z', 'A'..'Z', '0'..'9', '_', '$'} - -proc skipSqlTrivia(sql: string; pos: var int) = - var keepReading = true - while keepReading and pos < sql.len: - keepReading = false - while pos < sql.len and sql[pos] in Whitespace: - inc pos - keepReading = true - if pos + 1 < sql.len and sql[pos] == '-' and sql[pos + 1] == '-': - inc pos, 2 - while pos < sql.len and sql[pos] notin {'\c', '\L'}: - inc pos - keepReading = true - elif pos + 1 < sql.len and sql[pos] == '/' and sql[pos + 1] == '*': - inc pos, 2 - while pos + 1 < sql.len and not (sql[pos] == '*' and sql[pos + 1] == '/'): - inc pos - if pos + 1 < sql.len: - inc pos, 2 - keepReading = true - -proc consumeSqlKeyword(sql: string; pos: var int; keyword: string): bool = - skipSqlTrivia(sql, pos) - let finish = pos + keyword.len - if finish > sql.len: - return false - if cmpIgnoreCase(sql[pos ..< finish], keyword) != 0: - return false - if finish < sql.len and isSqlIdentChar(sql[finish]): - return false - pos = finish - true - -proc readSqlIdentPart(sql: string; pos: var int): string = - skipSqlTrivia(sql, pos) - if pos >= sql.len: - return "" - - if sql[pos] == '"': - inc pos - while pos < sql.len: - if sql[pos] == '"': - if pos + 1 < sql.len and sql[pos + 1] == '"': - result.add('"') - inc pos, 2 - else: - inc pos - return - else: - result.add(sql[pos]) - inc pos - else: - while pos < sql.len and isSqlIdentChar(sql[pos]): - result.add(sql[pos]) - inc pos - -proc readSqlQualifiedIdent(sql: string; pos: var int): string = - result = readSqlIdentPart(sql, pos) - if result.len == 0: - return - - while true: - let beforeDot = pos - skipSqlTrivia(sql, pos) - if pos >= sql.len or sql[pos] != '.': - pos = beforeDot - return - inc pos - let part = readSqlIdentPart(sql, pos) - if part.len == 0: - return - result.add('.') - result.add(part) - -proc readSqlStringLiteral(sql: string; pos: var int; value: var string): bool = - skipSqlTrivia(sql, pos) - if pos >= sql.len or sql[pos] != '\'': - return false - - inc pos - while pos < sql.len: - if sql[pos] == '\'': - if pos + 1 < sql.len and sql[pos + 1] == '\'': - value.add('\'') - inc pos, 2 - else: - inc pos - return true - else: - value.add(sql[pos]) - inc pos - -proc readDollarQuotedBody(sql: string; pos: var int; body: var string): bool = - if pos >= sql.len or sql[pos] != '$': - return false - - let delimiterStart = pos - inc pos - while pos < sql.len and sql[pos] in {'a'..'z', 'A'..'Z', '0'..'9', '_'}: - inc pos - if pos >= sql.len or sql[pos] != '$': - pos = delimiterStart - return false - - inc pos - let delimiter = sql[delimiterStart ..< pos] - let bodyEnd = sql.find(delimiter, pos) - if bodyEnd < 0: - pos = delimiterStart - return false - - body = sql[pos ..< bodyEnd] - pos = bodyEnd + delimiter.len - true - -proc readSqlEnumValues(sql: string; pos: var int): seq[string] = - skipSqlTrivia(sql, pos) - if pos >= sql.len or sql[pos] != '(': - return @[] - inc pos - - var done = false - while not done and pos < sql.len: - skipSqlTrivia(sql, pos) - if pos < sql.len and sql[pos] == ')': - inc pos - done = true - else: - var value = "" - if not readSqlStringLiteral(sql, pos, value): - done = true - else: - result.add(value) - skipSqlTrivia(sql, pos) - if pos < sql.len and sql[pos] == ',': - inc pos - elif pos < sql.len and sql[pos] == ')': - inc pos - done = true - else: - done = true - -proc registerEnum(enums: var KnownEnums; typeName: string; values: seq[string]) = - if typeName.len == 0 or values.len == 0: - return - - let normalized = typeName.toLowerAscii - enums[normalized] = values - let dotPos = normalized.rfind('.') - if dotPos >= 0 and dotPos + 1 < normalized.len: - enums[normalized[dotPos + 1 .. ^1]] = values - -proc collectEnumTypes(schemaSql: string; enums: var KnownEnums) = - var pos = 0 - while pos < schemaSql.len: - skipSqlTrivia(schemaSql, pos) - if pos >= schemaSql.len: - break - - case schemaSql[pos] - of '\'': - var ignored = "" - if not readSqlStringLiteral(schemaSql, pos, ignored): - inc pos - of '"': - discard readSqlIdentPart(schemaSql, pos) - of '$': - var body = "" - if readDollarQuotedBody(schemaSql, pos, body): - collectEnumTypes(body, enums) - else: - inc pos - else: - if schemaSql[pos] notin {'a'..'z', 'A'..'Z', '_'}: - inc pos - continue - - let wordStart = pos - while pos < schemaSql.len and isSqlIdentChar(schemaSql[pos]): - inc pos - if cmpIgnoreCase(schemaSql[wordStart ..< pos], "create") != 0: - continue - - var declarationPos = pos - if not consumeSqlKeyword(schemaSql, declarationPos, "type"): - continue - - let beforeOptional = declarationPos - if consumeSqlKeyword(schemaSql, declarationPos, "if"): - if not consumeSqlKeyword(schemaSql, declarationPos, "not") or - not consumeSqlKeyword(schemaSql, declarationPos, "exists"): - declarationPos = beforeOptional - - let typeName = readSqlQualifiedIdent(schemaSql, declarationPos) - if consumeSqlKeyword(schemaSql, declarationPos, "as") and - consumeSqlKeyword(schemaSql, declarationPos, "enum"): - enums.registerEnum(typeName, readSqlEnumValues(schemaSql, declarationPos)) - pos = max(pos, declarationPos) - proc collectEnumTypes(schemaSql: string): KnownEnums = result = initTable[string, seq[string]]() - collectEnumTypes(schemaSql, result) + let definitions = parseEnumTypeDefs(schemaSql) + for i in 0 ..< definitions.len: + let definition = definitions[i] + let typeName = sqlIdentName(definition[0]).toLowerAscii() + var values: seq[string] = @[] + for value in definition[1].sons: + values.add(value.strVal) + result[typeName] = values + + let dotPos = typeName.rfind('.') + if dotPos >= 0 and dotPos + 1 < typeName.len: + result[typeName[dotPos + 1 .. ^1]] = values proc getType(n: SqlNode; enums: KnownEnums): DbType = var it = n diff --git a/ormin/parsesql_tmp.nim b/ormin/parsesql_tmp.nim index d304fe9..8563f3e 100644 --- a/ormin/parsesql_tmp.nim +++ b/ormin/parsesql_tmp.nim @@ -2043,3 +2043,32 @@ proc parseSql*(input: string, filename = "", considerTypeParams = false): SqlNod ## `filename` is only used for error messages. ## Syntax errors raise an `SqlParseError` exception. parseSql(newStringStream(input), "", considerTypeParams) + +proc scanEnumTypeDefs(input, filename: string; definitions: SqlNode) = + var p: SqlParser + open(p, newStringStream(input), filename) + try: + while p.tok.kind != tkEof: + if p.tok.kind == tkDollarQuotedConstant: + let body = p.tok.literal + getTok(p) + scanEnumTypeDefs(body, filename, definitions) + elif isKeyw(p, "create"): + getTok(p) + if isKeyw(p, "type"): + let definition = parseIfNotExists(p, nkCreateType) + definition.add(parseQualifiedIdentifier(p)) + if isKeyw(p, "as"): + getTok(p) + if isKeyw(p, "enum"): + definition.add(parseDataType(p)) + definitions.add(definition) + else: + getTok(p) + finally: + close(p) + +proc parseEnumTypeDefs*(input: string; filename = ""): SqlNode = + ## Finds enum type definitions, including definitions inside dollar-quoted blocks. + result = newNode(nkStmtList) + scanEnumTypeDefs(input, filename, result)