fix(plugin-surrealdb): BETWEEN, REGEX and IS EMPTY filters become equality checks

CHANGELOG.md

@@ -46,6 +46,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - PostgreSQL `money` values written as null in Parquet exports.
 - Files left behind when a multi-table Parquet export is stopped between tables.
 - Filter-bar BETWEEN refused on Typesense and Weaviate, and given the wrong lower bound on BigQuery.
+- SurrealDB between, matches regex, is empty and raw filters run as equality, and is not empty showing only empty rows.
+- SurrealDB edits to `in` and `out` dropped without a word when the same row had another edit.
 - Cassandra filter error telling MCP clients to use a Match All control they do not have.
 - Japanese, Chinese and Korean text in CSV, TSV and SQL files opening as garbled characters.
 - Every row of a UTF-16 data file rewritten on save.

Plugins/SurrealDBDriverPlugin/SurrealDBPluginDriver.swift

@@ -264,23 +264,7 @@ final class SurrealDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable {
     func buildFilteredQuery(
         table: String,
         schema: String?,
-        filters: [(column: String, op: String, value: String)],
-        logicMode: String,
-        sortColumns: [(columnIndex: Int, ascending: Bool)],
-        columns: [String],
-        limit: Int,
-        offset: Int
-    ) -> String? {
-        buildFilteredQuery(
-            table: table, schema: schema, filters: filters, logicMode: logicMode,
-            sortColumns: sortColumns, columns: columns, limit: limit, offset: offset, columnKinds: [:]
-        )
-    }
-
-    func buildFilteredQuery(
-        table: String,
-        schema: String?,
-        filters: [(column: String, op: String, value: String)],
+        queryFilters: [PluginQueryFilter],
         logicMode: String,
         sortColumns: [(columnIndex: Int, ascending: Bool)],
         columns: [String],
@@ -291,7 +275,7 @@ final class SurrealDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable {
         SurrealQueryBuilder.filtered(
             table: table,
             scope: scope(forSchema: schema),
-            filters: filters,
+            filters: queryFilters,
             logicMode: logicMode,
             sortColumns: Self.sorts(sortColumns, columns: columns),
             limit: limit,
@@ -302,16 +286,25 @@ final class SurrealDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable {
 
     func fetchFilteredRowCount(
         table: String,
-        filters: [(column: String, op: String, value: String)],
+        queryFilters: [PluginQueryFilter],
+        logicMode: String
+    ) async throws -> Int? {
+        try await count(table: table, schema: nil, filters: queryFilters, logicMode: logicMode)
+    }
+
+    func fetchExactRowCount(
+        table: String,
+        schema: String?,
+        queryFilters: [PluginQueryFilter],
         logicMode: String
     ) async throws -> Int? {
-        try await count(table: table, schema: nil, filters: filters, logicMode: logicMode)
+        try await count(table: table, schema: schema, filters: queryFilters, logicMode: logicMode)
     }
 
     func count(
         table: String,
         schema: String?,
-        filters: [(column: String, op: String, value: String)],
+        filters: [PluginQueryFilter],
         logicMode: String
     ) async throws -> Int? {
         let scope = scope(forSchema: schema)

Plugins/SurrealDBDriverPlugin/SurrealQueryBuilder.swift

@@ -28,7 +28,31 @@ public struct SurrealScope: Equatable, Sendable {
     }
 }
 
+public struct SurrealFilterRefusal: Error, Equatable, Sendable {
+    public let message: String
+
+    public static let incompleteRange = SurrealFilterRefusal(
+        message: String(localized: "Enter both bounds to filter with BETWEEN.")
+    )
+
+    public static let rawConditionNotReadOnly = SurrealFilterRefusal(
+        message: String(
+            localized: "A raw SurrealDB filter must be one condition that only reads. Run anything else in the SurrealQL editor."
+        )
+    )
+
+    public static func unsupportedOperator(_ op: String) -> SurrealFilterRefusal {
+        SurrealFilterRefusal(message: String(format: String(localized: "SurrealDB cannot filter rows with %@."), op))
+    }
+
+    public var statement: String {
+        "THROW " + SurrealQL.stringLiteral(message) + ";"
+    }
+}
+
 public enum SurrealQueryBuilder {
+    public static let rawFilterColumn = "__RAW__"
+
     public static func browse(
         table: String,
         scope: SurrealScope,
@@ -42,32 +66,40 @@ public enum SurrealQueryBuilder {
     public static func filtered(
         table: String,
         scope: SurrealScope,
-        filters: [(column: String, op: String, value: String)],
+        filters: [PluginQueryFilter],
         logicMode: String,
         sortColumns: [(column: String, ascending: Bool)],
         limit: Int,
         offset: Int,
         columnKinds: [String: PluginColumnKind] = [:]
     ) -> String {
-        let clause = whereClause(filters: filters, logicMode: logicMode, columnKinds: columnKinds)
-        return compose(
-            scope: scope,
-            statement: select(table: table, where: clause, sortColumns: sortColumns, limit: limit, offset: offset)
-        )
+        do throws(SurrealFilterRefusal) {
+            let clause = try whereClause(filters: filters, logicMode: logicMode, columnKinds: columnKinds)
+            return compose(
+                scope: scope,
+                statement: select(table: table, where: clause, sortColumns: sortColumns, limit: limit, offset: offset)
+            )
+        } catch {
+            return compose(scope: scope, statement: error.statement)
+        }
     }
 
     public static func count(
         table: String,
         scope: SurrealScope,
-        filters: [(column: String, op: String, value: String)],
+        filters: [PluginQueryFilter],
         logicMode: String
     ) -> String {
-        var statement = "SELECT count() AS total FROM " + SurrealQL.quoteIdentifier(table)
-        if let clause = whereClause(filters: filters, logicMode: logicMode) {
-            statement += " WHERE " + clause
+        do throws(SurrealFilterRefusal) {
+            var statement = "SELECT count() AS total FROM " + SurrealQL.quoteIdentifier(table)
+            if let clause = try whereClause(filters: filters, logicMode: logicMode) {
+                statement += " WHERE " + clause
+            }
+            statement += " GROUP ALL;"
+            return compose(scope: scope, statement: statement)
+        } catch {
+            return compose(scope: scope, statement: error.statement)
         }
-        statement += " GROUP ALL;"
-        return compose(scope: scope, statement: statement)
     }
 
     public static func sample(table: String, scope: SurrealScope, limit: Int) -> String {
@@ -112,20 +144,27 @@ public enum SurrealQueryBuilder {
     }
 
     public static func whereClause(
-        filters: [(column: String, op: String, value: String)],
+        filters: [PluginQueryFilter],
         logicMode: String,
         columnKinds: [String: PluginColumnKind] = [:]
-    ) -> String? {
-        let conditions = filters.compactMap { condition($0, kind: columnKinds[$0.column]) }
+    ) throws(SurrealFilterRefusal) -> String? {
+        var conditions: [String] = []
+        for filter in filters {
+            guard let condition = try condition(filter, kind: columnKinds[filter.column]) else { continue }
+            conditions.append(condition)
+        }
         guard !conditions.isEmpty else { return nil }
         let separator = logicMode.lowercased() == "or" ? " OR " : " AND "
         return conditions.joined(separator: separator)
     }
 
     private static func condition(
-        _ filter: (column: String, op: String, value: String),
+        _ filter: PluginQueryFilter,
         kind: PluginColumnKind?
-    ) -> String? {
+    ) throws(SurrealFilterRefusal) -> String? {
+        if filter.column == rawFilterColumn {
+            return try SurrealRawCondition.parenthesized(filter.value)
+        }
         guard !filter.column.isEmpty else { return nil }
         let column = SurrealQL.quoteIdentifier(filter.column)
         let op = filter.op.uppercased().trimmingCharacters(in: .whitespaces)
@@ -136,6 +175,10 @@ public enum SurrealQueryBuilder {
             return "(\(column) = NONE OR \(column) = NULL)"
         case "IS NOT NULL":
             return "(\(column) != NONE AND \(column) != NULL)"
+        case "IS EMPTY":
+            return "(\(column) = NONE OR \(column) = NULL OR \(column) = '')"
+        case "IS NOT EMPTY":
+            return "(\(column) != NONE AND \(column) != NULL AND \(column) != '')"
         case "CONTAINS":
             return "string::contains(<string> \(column), \(SurrealQL.stringLiteral(value)))"
         case "NOT CONTAINS":
@@ -144,17 +187,56 @@ public enum SurrealQueryBuilder {
             return "string::starts_with(<string> \(column), \(SurrealQL.stringLiteral(value)))"
         case "ENDS WITH":
             return "string::ends_with(<string> \(column), \(SurrealQL.stringLiteral(value)))"
+        case "REGEX":
+            let match = "string::matches(<string> \(column), \(SurrealQL.stringLiteral(value)))"
+            return "(\(column) != NONE AND \(column) != NULL AND \(match))"
         case "IN":
             return "\(column) INSIDE \(listLiteral(value, kind: kind))"
         case "NOT IN":
             return "\(column) NOTINSIDE \(listLiteral(value, kind: kind))"
+        case "BETWEEN":
+            let bounds = try rangeBounds(filter)
+            let lower = literal(bounds.lower, kind: kind)
+            let upper = literal(bounds.upper, kind: kind)
+            return "(\(column) >= \(lower) AND \(column) <= \(upper))"
         case "=", "!=", ">", ">=", "<", "<=":
             return "\(column) \(op) \(literal(value, kind: kind))"
         case "LIKE":
             return "string::contains(<string> \(column), \(SurrealQL.stringLiteral(unwrapWildcards(value))))"
         default:
-            return "\(column) = \(literal(value, kind: kind))"
+            throw SurrealFilterRefusal.unsupportedOperator(filter.op)
+        }
+    }
+
+    private static func rangeBounds(
+        _ filter: PluginQueryFilter
+    ) throws(SurrealFilterRefusal) -> (lower: String, upper: String) {
+        if let upper = filter.secondValue {
+            return try completeRange(lower: lowerBound(of: filter.value, upperBound: upper), upper: upper)
         }
+        let scalars = filter.value.unicodeScalars
+        guard let separator = scalars.firstIndex(of: ",") else { throw .incompleteRange }
+        return try completeRange(
+            lower: String(scalars[..<separator]),
+            upper: String(scalars[scalars.index(after: separator)...])
+        )
+    }
+
+    private static func lowerBound(of joinedValue: String, upperBound: String) -> String {
+        let joinedSuffix = ("," + upperBound).unicodeScalars
+        let scalars = joinedValue.unicodeScalars
+        guard scalars.reversed().starts(with: joinedSuffix.reversed()) else { return joinedValue }
+        return String(scalars.dropLast(joinedSuffix.count))
+    }
+
+    private static func completeRange(
+        lower: String,
+        upper: String
+    ) throws(SurrealFilterRefusal) -> (lower: String, upper: String) {
+        let lowerBound = lower.trimmingCharacters(in: .whitespaces)
+        let upperBound = upper.trimmingCharacters(in: .whitespaces)
+        guard !lowerBound.isEmpty, !upperBound.isEmpty else { throw .incompleteRange }
+        return (lowerBound, upperBound)
     }
 
     private static func listLiteral(_ value: String, kind: PluginColumnKind?) -> String {

Plugins/SurrealDBDriverPlugin/SurrealRawCondition.swift

@@ -0,0 +1,135 @@
+//
+//  SurrealRawCondition.swift
+//  SurrealDBDriverPlugin
+//
+
+import Foundation
+
+internal enum SurrealRawCondition {
+    static func parenthesized(_ text: String) throws(SurrealFilterRefusal) -> String? {
+        let condition = text.trimmingCharacters(in: .whitespacesAndNewlines)
+        guard !condition.isEmpty else { return nil }
+        guard onlyReads(Array(condition.unicodeScalars)) else { throw .rawConditionNotReadOnly }
+        return "(" + condition + ")"
+    }
+
+    private static let writingKeywords: Set<String> = [
+        "ACCESS", "ALTER", "CREATE", "DEFINE", "DELETE", "INSERT", "KILL", "LIVE", "OPTION",
+        "REBUILD", "RELATE", "REMOVE", "UPDATE", "UPSERT", "USE"
+    ]
+
+    private static let writingFunctionNamespaces: Set<String> = ["api", "file", "fn", "http", "sequence"]
+
+    private static func onlyReads(_ scalars: [Unicode.Scalar]) -> Bool {
+        var index = 0
+        var depth = 0
+        while index < scalars.count {
+            let scalar = scalars[index]
+            if let closing = closingDelimiter(of: scalar) {
+                guard let end = endOfQuoted(scalars, from: index + 1, closing: closing) else { return false }
+                index = end + 1
+                continue
+            }
+            if isWordScalar(scalar) {
+                let end = endOfWord(scalars, from: index)
+                guard wordOnlyReads(scalars, start: index, end: end) else { return false }
+                index = end
+                continue
+            }
+            guard !startsStatementOrComment(scalars, at: index) else { return false }
+            if scalar == "(" {
+                depth += 1
+            } else if scalar == ")" {
+                depth -= 1
+                guard depth >= 0 else { return false }
+            }
+            index += 1
+        }
+        return depth == 0
+    }
+
+    private static func closingDelimiter(of scalar: Unicode.Scalar) -> Unicode.Scalar? {
+        switch scalar {
+        case "'", "\"", "`":
+            return scalar
+        case "\u{27E8}":
+            return "\u{27E9}"
+        default:
+            return nil
+        }
+    }
+
+    private static func endOfQuoted(_ scalars: [Unicode.Scalar], from start: Int, closing: Unicode.Scalar) -> Int? {
+        var index = start
+        while index < scalars.count {
+            if scalars[index] == "\\" {
+                index += 2
+                continue
+            }
+            if scalars[index] == closing {
+                return index
+            }
+            index += 1
+        }
+        return nil
+    }
+
+    private static func isWordScalar(_ scalar: Unicode.Scalar) -> Bool {
+        scalar == "_" || CharacterSet.alphanumerics.contains(scalar)
+    }
+
+    private static func endOfWord(_ scalars: [Unicode.Scalar], from start: Int) -> Int {
+        var index = start
+        while index < scalars.count, isWordScalar(scalars[index]) {
+            index += 1
+        }
+        return index
+    }
+
+    private static func wordOnlyReads(_ scalars: [Unicode.Scalar], start: Int, end: Int) -> Bool {
+        let word = String(String.UnicodeScalarView(scalars[start..<end]))
+        if isFollowedByPathSeparator(scalars, at: end), writingFunctionNamespaces.contains(word.lowercased()) {
+            return false
+        }
+        if namesParameterFieldOrFunction(scalars, start: start) {
+            return true
+        }
+        return !writingKeywords.contains(word.uppercased())
+    }
+
+    private static func namesParameterFieldOrFunction(_ scalars: [Unicode.Scalar], start: Int) -> Bool {
+        guard start > 0 else { return false }
+        switch scalars[start - 1] {
+        case "$":
+            return true
+        case ".":
+            return start >= 2 && scalars[start - 2] != "."
+        case ":":
+            return start >= 3 && scalars[start - 2] == ":" && isWordScalar(scalars[start - 3])
+        default:
+            return false
+        }
+    }
+
+    private static func isFollowedByPathSeparator(_ scalars: [Unicode.Scalar], at index: Int) -> Bool {
+        var next = index
+        while next < scalars.count, CharacterSet.whitespacesAndNewlines.contains(scalars[next]) {
+            next += 1
+        }
+        return next + 1 < scalars.count && scalars[next] == ":" && scalars[next + 1] == ":"
+    }
+
+    private static func startsStatementOrComment(_ scalars: [Unicode.Scalar], at index: Int) -> Bool {
+        let next = index + 1 < scalars.count ? scalars[index + 1] : nil
+        switch scalars[index] {
+        case ";", "#":
+            return true
+        case "-":
+            return next == "-"
+        case "/":
+            return next == "/" || next == "*"
+        default:
+            return false
+        }
+    }
+}