From 69c2b1c6d374324cedd850fd14d1f203101984ee Mon Sep 17 00:00:00 2001 From: Alsey Coleman Miller Date: Thu, 16 Jul 2026 14:09:02 -0400 Subject: [PATCH 1/3] Register custom functions via the SQLite C API directly Call sqlite3_create_function_v2 with @convention(c) function pointers and a retained context pointer, instead of SQLite.swift's createFunction. SQLite.swift registers the callback as a @convention(block) closure cast to a raw pointer, which is unreliable off Apple platforms and segfaults on Linux (upstream stephencelis/SQLite.swift#1071). A plain C function pointer works everywhere. - New CustomFunction.swift: C-API registration, argument/result marshalling between sqlite3_value/context and AttributeValue, with the SQLite C module imported from the system SQLite3 on Apple and the embedded copy elsewhere. - Package.swift depends on stephencelis/CSQLite so the C API is available on Linux/Android where the system SQLite3 module isn't. - Removed the Darwin-only gate on the custom-function tests; they now run on every platform. --- Package.swift | 16 ++- Sources/CoreModelSQLite/AttributeValue.swift | 20 ---- Sources/CoreModelSQLite/CustomFunction.swift | 113 ++++++++++++++++++ Sources/CoreModelSQLite/Database.swift | 23 +--- Sources/CoreModelSQLite/Error.swift | 3 + Sources/CoreModelSQLite/ViewContext.swift | 2 +- .../CustomFunctionTests.swift | 9 -- 7 files changed, 133 insertions(+), 53 deletions(-) create mode 100644 Sources/CoreModelSQLite/CustomFunction.swift diff --git a/Package.swift b/Package.swift index 54043cb..7b9ac32 100644 --- a/Package.swift +++ b/Package.swift @@ -36,7 +36,14 @@ let package = Package( url: "https://github.com/PureSwift/CoreModel", from: "2.8.0" ), - sqliteDependency + sqliteDependency, + // Off Apple platforms, SQLite.swift links the embedded SQLite from this package. + // We depend on it directly so we can call the SQLite C API (custom functions) + // where the system `SQLite3` module isn't available. + .package( + url: "https://github.com/stephencelis/CSQLite", + from: "3.50.4" + ) ], targets: [ .target( @@ -46,6 +53,13 @@ let package = Package( .product( name: "SQLite", package: "SQLite.swift" + ), + // On Apple platforms the SQLite C API comes from the system `SQLite3` + // module; elsewhere it comes from SQLite.swift's embedded copy. + .product( + name: "SQLiteSwiftCSQLite", + package: "CSQLite", + condition: .when(platforms: [.linux, .android]) ) ] ), diff --git a/Sources/CoreModelSQLite/AttributeValue.swift b/Sources/CoreModelSQLite/AttributeValue.swift index 0471189..4681e5d 100644 --- a/Sources/CoreModelSQLite/AttributeValue.swift +++ b/Sources/CoreModelSQLite/AttributeValue.swift @@ -43,26 +43,6 @@ internal extension AttributeValue { } } - /// Decode from a SQLite binding value with no declared attribute type (e.g. a - /// raw argument passed into a custom SQL function), inferring the value's shape - /// from the binding's runtime type. - init(binding: Binding?) { - switch binding { - case .none: - self = .null - case let value as Int64: - self = .int64(value) - case let value as Double: - self = .double(value) - case let value as String: - self = .string(value) - case let value as Blob: - self = .data(Data(value.bytes)) - default: - self = .null - } - } - /// Decode from a SQLite binding value, interpreting it according to the declared attribute type. init(binding: Binding?, type: AttributeType) throws { guard let binding else { diff --git a/Sources/CoreModelSQLite/CustomFunction.swift b/Sources/CoreModelSQLite/CustomFunction.swift new file mode 100644 index 0000000..4019ebb --- /dev/null +++ b/Sources/CoreModelSQLite/CustomFunction.swift @@ -0,0 +1,113 @@ +// +// CustomFunction.swift +// CoreModel-SQLite +// +// Created by Alsey Coleman Miller on 7/16/26. +// + +import Foundation +import CoreModel +import SQLite +#if canImport(Darwin) +import SQLite3 +#elseif canImport(SQLiteSwiftCSQLite) +import SQLiteSwiftCSQLite +#elseif canImport(CSQLite) +import CSQLite +#else +import SQLite3 +#endif + +// Registers custom scalar functions by calling `sqlite3_create_function_v2` directly +// with `@convention(c)` function pointers, rather than SQLite.swift's `createFunction`. +// SQLite.swift registers the callback with a `@convention(block)` closure cast to a raw +// pointer, which is unreliable off Apple platforms (upstream +// https://github.com/stephencelis/SQLite.swift/issues/1071). A plain C function pointer +// plus a retained context pointer works identically on every platform. + +/// The SQLite `SQLITE_TRANSIENT` sentinel destructor, telling SQLite to copy a result +/// value immediately (it is a macro in C, so it isn't imported). +private let transientDestructor = unsafeBitCast(-1, to: sqlite3_destructor_type.self) + +/// Retains a ``DatabaseFunction`` so it can be passed through SQLite as an opaque pointer +/// and recovered inside the C callback. +private final class FunctionBox { + let function: DatabaseFunction + init(_ function: DatabaseFunction) { self.function = function } +} + +internal extension SQLite.Connection { + + /// Registers a `DatabaseFunction` with this connection via the SQLite C API. + func register(function: DatabaseFunction) throws { + let box = Unmanaged.passRetained(FunctionBox(function)) + let flags = SQLITE_UTF8 | (function.deterministic ? SQLITE_DETERMINISTIC : 0) + let argumentCount = function.argumentCount.map { Int32($0) } ?? -1 + let code = sqlite3_create_function_v2( + handle, + function.name, + argumentCount, + flags, + box.toOpaque(), + { context, argc, argv in + let function = Unmanaged.fromOpaque(sqlite3_user_data(context)).takeUnretainedValue().function + var arguments = [AttributeValue?]() + arguments.reserveCapacity(Int(argc)) + for index in 0...fromOpaque(pointer).release() + } + ) + guard code == SQLITE_OK else { + box.release() // xDestroy isn't called when registration fails + throw SQLiteDatabaseError.unableToCreateFunction(function.name, code) + } + } +} + +/// Read a SQLite argument value into an ``AttributeValue``, inferring its shape from the +/// value's runtime storage class. +private func argumentValue(_ value: OpaquePointer?) -> AttributeValue { + switch sqlite3_value_type(value) { + case SQLITE_INTEGER: + return .int64(sqlite3_value_int64(value)) + case SQLITE_FLOAT: + return .double(sqlite3_value_double(value)) + case SQLITE_TEXT: + guard let text = sqlite3_value_text(value) else { return .null } + return .string(String(cString: text)) + case SQLITE_BLOB: + guard let bytes = sqlite3_value_blob(value) else { return .data(Data()) } + return .data(Data(bytes: bytes, count: Int(sqlite3_value_bytes(value)))) + default: + return .null + } +} + +/// Set a function's result on the SQLite context from an ``AttributeValue``. +private func setResult(_ context: OpaquePointer?, _ value: AttributeValue?) { + guard let value, let binding = value.binding else { + sqlite3_result_null(context) + return + } + switch binding { + case let integer as Int64: + sqlite3_result_int64(context, integer) + case let double as Double: + sqlite3_result_double(context, double) + case let text as String: + sqlite3_result_text(context, text, -1, transientDestructor) + case let blob as Blob: + sqlite3_result_blob(context, blob.bytes, Int32(blob.bytes.count), transientDestructor) + default: + sqlite3_result_null(context) + } +} diff --git a/Sources/CoreModelSQLite/Database.swift b/Sources/CoreModelSQLite/Database.swift index 593fc65..5be7b4e 100644 --- a/Sources/CoreModelSQLite/Database.swift +++ b/Sources/CoreModelSQLite/Database.swift @@ -114,13 +114,8 @@ extension SQLiteDatabase: ModelStorage { /// Registers a custom scalar function so it can be invoked from a predicate or sort /// descriptor via ``FetchRequest/Predicate/Expression/function(_:)``. - /// - /// - Important: Only supported on Apple platforms. The underlying SQLite.swift - /// `createFunction` registers the callback through `@convention(block)` + - /// `unsafeBitCast`, which is unreliable on non-Apple platforms and can corrupt - /// SQLite's heap (upstream https://github.com/stephencelis/SQLite.swift/issues/1071). public func register(function: DatabaseFunction) async throws { - connection.register(function: function) + try connection.register(function: function) } } @@ -135,22 +130,6 @@ public extension SQLiteDatabase { } } -internal extension SQLite.Connection { - - /// Registers a `DatabaseFunction` with this connection, bridging SQLite's untyped - /// `Binding` values to/from `AttributeValue` at the boundary. - func register(function: DatabaseFunction) { - createFunction( - function.name, - argumentCount: function.argumentCount.map { UInt($0) }, - deterministic: function.deterministic - ) { arguments in - let values: [AttributeValue?] = arguments.map { AttributeValue(binding: $0) } - return function.evaluate(values)?.binding - } - } -} - internal extension SQLiteDatabase { func asyncYield() async throws { diff --git a/Sources/CoreModelSQLite/Error.swift b/Sources/CoreModelSQLite/Error.swift index 64a5082..b122765 100644 --- a/Sources/CoreModelSQLite/Error.swift +++ b/Sources/CoreModelSQLite/Error.swift @@ -20,4 +20,7 @@ public enum SQLiteDatabaseError: Error { /// The predicate cannot be represented as SQL. case invalidPredicate(FetchRequest.Predicate) + + /// A custom function could not be registered with SQLite. Carries the SQLite result code. + case unableToCreateFunction(String, Int32) } diff --git a/Sources/CoreModelSQLite/ViewContext.swift b/Sources/CoreModelSQLite/ViewContext.swift index ff70182..9af2b0d 100644 --- a/Sources/CoreModelSQLite/ViewContext.swift +++ b/Sources/CoreModelSQLite/ViewContext.swift @@ -61,6 +61,6 @@ public final class SQLiteViewContext: ViewContext { /// paired ``SQLiteDatabase`` must also be registered here to be usable from /// queries run through this view context. public func register(function: DatabaseFunction) throws { - connection.register(function: function) + try connection.register(function: function) } } diff --git a/Tests/CoreModelSQLiteTests/CustomFunctionTests.swift b/Tests/CoreModelSQLiteTests/CustomFunctionTests.swift index d8235e3..ea77304 100644 --- a/Tests/CoreModelSQLiteTests/CustomFunctionTests.swift +++ b/Tests/CoreModelSQLiteTests/CustomFunctionTests.swift @@ -4,14 +4,6 @@ import CoreModel import SQLite @testable import CoreModelSQLite -// Custom SQL functions are only exercised on Apple platforms: SQLite.swift's -// `createFunction` registers the callback with `@convention(block)` + an -// `unsafeBitCast` to a raw pointer, which is unreliable on non-Apple platforms -// (the block pointer is invalid and corrupts SQLite's heap — upstream -// https://github.com/stephencelis/SQLite.swift/issues/1071). Gate these tests to -// Darwin so CI stays green until that is resolved. -#if canImport(Darwin) - /// A Haversine distance function, in meters, written directly in the test — CoreModelSQLite /// itself has no notion of "distance" or geo data; this exercises the generic /// `DatabaseFunction`/`.function` expression mechanism using a realistic example. @@ -323,4 +315,3 @@ private func randomSites(count: Int, seed: UInt64) -> [(id: ObjectID, latitude: #expect(oracle.isEmpty == false) } -#endif // canImport(Darwin) From 5e225479421789110eb20b179e53e391ac73428d Mon Sep 17 00:00:00 2001 From: Alsey Coleman Miller Date: Thu, 16 Jul 2026 14:25:31 -0400 Subject: [PATCH 2/3] Skip app-managed R*Tree test where the rtree module is unavailable R*Tree is an optional SQLite compile-time feature (SQLITE_ENABLE_RTREE), present in the system SQLite on Apple platforms but not in the embedded SQLite used on Linux. Gate the app-managed R*Tree test on a runtime capability probe so it runs where rtree exists and is skipped otherwise. The custom-function tests themselves now pass on every platform via the C-API registration. --- .../CustomFunctionTests.swift | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/Tests/CoreModelSQLiteTests/CustomFunctionTests.swift b/Tests/CoreModelSQLiteTests/CustomFunctionTests.swift index ea77304..0a05ac8 100644 --- a/Tests/CoreModelSQLiteTests/CustomFunctionTests.swift +++ b/Tests/CoreModelSQLiteTests/CustomFunctionTests.swift @@ -60,6 +60,18 @@ private struct SeededGenerator: RandomNumberGenerator { } } +/// Whether this SQLite build includes the R*Tree module (an optional compile-time +/// feature), determined by attempting to create an R*Tree virtual table in memory. +private let isRTreeAvailable: Bool = { + guard let connection = try? Connection(.inMemory) else { return false } + do { + try connection.execute("CREATE VIRTUAL TABLE rtree_probe USING rtree(id, minX, maxX)") + return true + } catch { + return false + } +}() + private func distanceSort(ascending: Bool = true) -> FetchRequest.SortDescriptor { .init(term: .function(.init(name: "distance", arguments: [ .keyPath("latitude"), .keyPath("longitude"), @@ -243,7 +255,12 @@ private func randomSites(count: Int, seed: UInt64) -> [(id: ObjectID, latitude: /// applies the exact `distance` function for correctness. The final result must match /// the brute-force in-memory oracle, and the prefilter must be sound (a superset of the /// exact matches) and actually prune. -@Test func appManagedRTreePrefilter() async throws { +/// +/// R*Tree is an optional SQLite compile-time feature (`SQLITE_ENABLE_RTREE`). It's present +/// in the system SQLite on Apple platforms but not in the embedded SQLite used elsewhere, +/// so this test is skipped where the `rtree` module is unavailable. +@Test(.enabled(if: isRTreeAvailable, "R*Tree module not compiled into this SQLite build")) +func appManagedRTreePrefilter() async throws { let path = temporaryDatabasePath(named: "GeoRTree") let database = try SQLiteDatabase(path: path, model: geoModel) try await database.register(function: distanceFunction) From 167304a9b40ef6b6045c4b459b2eafde6c573e99 Mon Sep 17 00:00:00 2001 From: Alsey Coleman Miller Date: Thu, 16 Jul 2026 18:00:05 -0400 Subject: [PATCH 3/3] Add distance query benchmarks Compare distance filtering/sorting via the registered SQL `distance` function (executed in SQLite) against fetching every row and filtering/sorting in Swift. Two scenarios: a selective radius filter+sort, and a full sort with no filter. Opt-in via the BENCHMARK environment variable so the 100k-row dataset doesn't slow the normal test matrix: BENCHMARK=1 swift test -c release --filter Benchmark --- .../DistancePerformanceTests.swift | 161 ++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 Tests/CoreModelSQLiteTests/DistancePerformanceTests.swift diff --git a/Tests/CoreModelSQLiteTests/DistancePerformanceTests.swift b/Tests/CoreModelSQLiteTests/DistancePerformanceTests.swift new file mode 100644 index 0000000..9870740 --- /dev/null +++ b/Tests/CoreModelSQLiteTests/DistancePerformanceTests.swift @@ -0,0 +1,161 @@ +// +// DistancePerformanceTests.swift +// CoreModel-SQLite +// +// Compares distance filtering/sorting via a registered SQL function (executed in +// SQLite) against fetching every row and filtering/sorting in Swift. +// +// These are benchmarks, not correctness tests (though they assert the two paths +// agree). They insert a large dataset and are skipped unless `BENCHMARK` is set in +// the environment, so they don't slow the normal test matrix. Run with: +// +// BENCHMARK=1 swift test -c release --filter Benchmark +// + +import Foundation +import Testing +import CoreModel +import SQLite +@testable import CoreModelSQLite + +/// Benchmarks are opt-in: set `BENCHMARK` in the environment to run them. +private let benchmarksEnabled = ProcessInfo.processInfo.environment["BENCHMARK"] != nil + +private let benchmarkReferenceLatitude = 35.7796 +private let benchmarkReferenceLongitude = -78.6382 +private let benchmarkRowCount = 100_000 +private let benchmarkIterations = 10 + +private func benchmarkHaversine(_ lat1: Double, _ lon1: Double, _ lat2: Double, _ lon2: Double) -> Double { + let earthRadius = 6_371_000.0 + let dLat = (lat2 - lat1) * .pi / 180 + let dLon = (lon2 - lon1) * .pi / 180 + let a = sin(dLat / 2) * sin(dLat / 2) + + cos(lat1 * .pi / 180) * cos(lat2 * .pi / 180) * sin(dLon / 2) * sin(dLon / 2) + return earthRadius * 2 * atan2(a.squareRoot(), (1 - a).squareRoot()) +} + +private let benchmarkDistanceFunction = DatabaseFunction(name: "distance", argumentCount: 4) { arguments in + guard case let .double(lat1) = arguments[0], case let .double(lon1) = arguments[1], + case let .double(lat2) = arguments[2], case let .double(lon2) = arguments[3] else { return nil } + return .double(benchmarkHaversine(lat1, lon1, lat2, lon2)) +} + +/// A database seeded with `benchmarkRowCount` sites spread across the continental US, +/// with the `distance` function registered. +private func makeBenchmarkDatabase() async throws -> SQLiteDatabase { + let model = Model(entities: [ + EntityDescription(id: "Site", attributes: [ + .init(id: "latitude", type: .double), + .init(id: "longitude", type: .double) + ], relationships: []) + ]) + let database = try SQLiteDatabase(path: temporaryDatabasePath(named: "Benchmark"), model: model) + try await database.register(function: benchmarkDistanceFunction) + + var rng = SystemRandomNumberGenerator() + var rows = [ModelData]() + rows.reserveCapacity(benchmarkRowCount) + for index in 0.. FetchRequest.SortDescriptor { + .init(term: .function(.init(name: "distance", arguments: [ + .keyPath("latitude"), .keyPath("longitude"), + .attribute(.double(benchmarkReferenceLatitude)), .attribute(.double(benchmarkReferenceLongitude)) + ])), ascending: true) +} + +private func benchmarkDistanceExpression() -> FetchRequest.Predicate.Expression { + .function(.init(name: "distance", arguments: [ + .keyPath("latitude"), .keyPath("longitude"), + .attribute(.double(benchmarkReferenceLatitude)), .attribute(.double(benchmarkReferenceLongitude)) + ])) +} + +/// Filter+sort by distance in Swift over already-fetched rows. +private func benchmarkInMemory(_ rows: [ModelData], radius: Double) -> [ObjectID] { + rows.compactMap { row -> (ObjectID, Double)? in + guard case let .double(latitude)? = row.attributes["latitude"], + case let .double(longitude)? = row.attributes["longitude"] else { return nil } + let distance = benchmarkHaversine(latitude, longitude, benchmarkReferenceLatitude, benchmarkReferenceLongitude) + return distance <= radius ? (row.id, distance) : nil + } + .sorted { $0.1 < $1.1 } + .map(\.0) +} + +@Suite(.enabled(if: benchmarksEnabled, "set BENCHMARK in the environment to run")) +struct DistancePerformanceBenchmarks { + + /// Selective radius filter + distance sort: SQLite returns only the matches, so the + /// custom-function path avoids materializing the excluded rows. + @Test func benchmarkSelectiveFilterAndSort() async throws { + let database = try await makeBenchmarkDatabase() + let radius = 500_000.0 // 500 km + let sqlRequest = FetchRequest( + entity: "Site", + sortDescriptors: [benchmarkDistanceSort()], + predicate: .comparison(.init(left: benchmarkDistanceExpression(), right: .attribute(.double(radius)), type: .lessThanOrEqualTo)) + ) + let allRequest = FetchRequest(entity: "Site") + + #expect(try await database.fetch(sqlRequest).count == benchmarkInMemory(try await database.fetch(allRequest), radius: radius).count) + + let clock = ContinuousClock() + var sqlMatches = 0 + let sqlStart = clock.now + for _ in 0..