feat: Decision 3.0 (d3-lite/nano/mini) Core ML runtime

Package.swift

@@ -72,6 +72,7 @@ let package = Package(
         .executableTarget(name: "ClefCompareDemo", dependencies: ["FluidUse"], exclude: ["README.md", "demo.sh"]),
         .executableTarget(name: "InternDecisionCheck", dependencies: ["FluidUse"]),
         .executableTarget(name: "Decision2Check", dependencies: ["FluidUse"]),
+        .executableTarget(name: "D3Check", dependencies: ["FluidUse"]),
         .executableTarget(
             name: "IssueTriageDemo", dependencies: ["FluidUse"], exclude: ["README.md", "fetch-issues.sh"]),
         .executableTarget(name: "Vela2Check", dependencies: ["FluidUse"]),

README.md

@@ -245,6 +245,44 @@ packed tokens) 63 / 113 ms. The upstream PyTorch runtime on the same Mac (MPS, f
 the Swift host against the Python reference. [Sources/IssueTriageDemo](Sources/IssueTriageDemo/README.md) labels 1,000
 GitHub issues live with Kai (type, owning team, priority, flags: five decisions per issue in one call).
 
+## Decision 3.0
+
+`D3Manager` runs vLLM Semantic Router's [Decision 3.0](https://huggingface.co/collections/vllm-sr/decision-30) d3
+models (Apache-2.0, v3.1.0) on the GPU through Core ML: d3-lite 0.85B, d3-nano 2.2B and d3-mini 4.5B (Qwen3.5 hybrid
+with a vision tower). A request is a state (text or JSON), optional images, and any number of named questions (choice,
+yes/no, score); as upstream, each question is its own row and gets a probability for every option. Images go through
+the fp32 vision tower once per request (the clef-vision preprocessor and host math), and the text graph (fp16 compute,
+int8 weights) runs per question in the smallest of three length buckets. Answers vs the upstream PyTorch runtime:
+
+| | text (100 q) | images (24–48 q) | video (28 answers, Python host) | per text question (M5 Pro) |
+|---|---|---|---|---|
+| d3-lite | 2 flips | 1 flip | 0 flips | ~46 ms |
+| d3-nano | 0 flips | 0 flips | 1 flip | ~65 ms |
+| d3-mini | 1 flip | 0 flips | 0 flips | ~160 ms |
+
+The pinned snapshots download from [FluidInference/d3-coreml](https://huggingface.co/FluidInference/d3-coreml) (`lite/`
+3.2 GB, `nano/` 9.1 GB, `mini/` 16 GB) on first use (macOS 15 / iOS 18).
+
+```swift
+let model = try await D3Manager.load(from: try await D3ModelStore.ensure(.lite))
+try await model.warm()
+let answers = try await model.answer(
+    state: .string("The order arrived damaged yesterday. The customer has a receipt and asks for a replacement today."),
+    images: [],  // or [try await model.encode(cgImage)]
+    questions: [
+        ("route", .choice("Which team should handle this request?", options: [
+            ("returns", "Refunds, replacements and damaged deliveries"), ("billing", "Payments, invoices and charges"),
+        ])),
+        ("receipt", .noul("Does the customer have a receipt?")),
+        ("urgency", .score("How urgent is this request?", levels: ["Routine", "Soon", "Today"])),
+    ])
+print(answers[0].best, answers[1].yes, answers[2].score)
+```
+
+That request takes 139 ms with d3-lite on an M5 Pro. `swift run -c release D3Check example lite|nano|mini` downloads a
+snapshot and answers it; `D3Check parity <model directory> <fixtures dir>` checks the Swift host against the Python
+Core ML host on marked browser screenshots (0 token mismatches, 0 flips).
+
 ## Vela 2.0 routing, safety and spans
 
 `Vela2Manager` runs vLLM Semantic Router × KR Labs'

Sources/D3Check/main.swift

@@ -0,0 +1,122 @@
+import CoreGraphics
+import FluidUse
+import Foundation
+import ImageIO
+
+/// d3 checks.
+///
+///     swift run -c release D3Check example lite|nano|mini     (downloads the pinned snapshot, answers the card request)
+///     swift run -c release D3Check parity <model dir> <fixtures dir>   (marked screenshots vs the Python Core ML host)
+///
+/// Fixtures: model-lab models/d3-demos/browser/make_swift_fixtures.py (token ids + probabilities per screenshot).
+@main
+struct D3Check {
+    struct VisionFixtures: Decodable { let fixtures: [VisionFixture] }
+
+    struct VisionFixture: Decodable {
+        struct Option: Decodable {
+            let key: String
+            let description: String
+        }
+        let image: String
+        let state: String
+        let instructions: String
+        let options: [Option]
+        let ids: [Int]
+        let probabilities: [Double]
+    }
+
+    @available(macOS 15.0, *)
+    static func parity(directory: URL, fixtures folder: URL) async throws {
+        let fixtures = try JSONDecoder().decode(
+            VisionFixtures.self, from: Data(contentsOf: folder.appendingPathComponent("fixtures.json"))
+        ).fixtures
+        let started = Date()
+        let manager = try await D3Manager.load(from: directory)
+        print(String(format: "loaded in %.1f s", Date().timeIntervalSince(started)))
+        var worst = 0.0
+        var mismatches = 0
+        var flips = 0
+        for (index, fixture) in fixtures.enumerated() {
+            guard let source = CGImageSourceCreateWithURL(folder.appendingPathComponent(fixture.image) as CFURL, nil),
+                let image = CGImageSourceCreateImageAtIndex(source, 0, nil)
+            else { throw D3Error.invalidAsset(fixture.image) }
+            let question = D3Question.choice(
+                fixture.instructions, options: fixture.options.map { ($0.key, $0.description) })
+            for pass in 0..<(index == 0 ? 2 : 1) {  // the first call per bucket also pays the GPU setup
+                let encoded = try await manager.encode(image)
+                let ids = try await manager.tokens(state: .string(fixture.state), question: question, images: [encoded])
+                let answer = try await manager.answer(
+                    state: .string(fixture.state), images: [encoded], questions: [("click", question)])[0]
+                guard pass == (index == 0 ? 1 : 0) else { continue }
+                if ids != fixture.ids { mismatches += 1 }
+                let d = zip(answer.probabilities, fixture.probabilities).map { abs($0 - $1) }.max() ?? 0
+                worst = max(worst, d)
+                let theirs = fixture.probabilities.indices.max { fixture.probabilities[$0] < fixture.probabilities[$1] }
+                let ours = answer.probabilities.indices.max { answer.probabilities[$0] < answer.probabilities[$1] }
+                if theirs != ours { flips += 1 }
+                print(
+                    String(
+                        format: "%@: %d tokens (%@), choice %@, max|dp| %.4f, vision %.0f ms + text %.0f ms",
+                        fixture.image, ids.count,
+                        ids == fixture.ids ? "ids match" : "IDS DIFFER (\(fixture.ids.count))",
+                        answer.best, d, encoded.milliseconds, answer.milliseconds))
+            }
+        }
+        print(
+            "\(fixtures.count) screenshots: token mismatches \(mismatches), flips \(flips), max|dp| \(String(format: "%.4f", worst))"
+        )
+        if mismatches > 0 || flips > 0 { exit(1) }
+    }
+
+    @available(macOS 15.0, *)
+    static func example(_ model: D3Model) async throws {
+        let directory = try await D3ModelStore.ensure(model) { file, bytes in
+            if bytes > 0 { print("downloaded \(file) (\(bytes / 1_000_000) MB)") }
+        }
+        let manager = try await D3Manager.load(from: directory)
+        let state = OrderedJSON.string(
+            "The order arrived damaged yesterday. The customer has a receipt and asks for a replacement today.")
+        let questions: [(id: String, question: D3Question)] = [
+            (
+                "route",
+                .choice(
+                    "Which team should handle this request?",
+                    options: [
+                        ("returns", "Refunds, replacements and damaged deliveries"),
+                        ("billing", "Payments, invoices and charges"), ("technical", "Product setup and faults"),
+                    ])
+            ),
+            ("receipt", .noul("Does the customer have a receipt?")),
+            ("urgency", .score("How urgent is this request?", levels: ["Routine", "Soon", "Today"])),
+        ]
+        _ = try await manager.answer(state: state, images: [], questions: questions)  // first call pays the setup
+        let started = Date()
+        let answers = try await manager.answer(state: state, images: [], questions: questions)
+        let ms = Date().timeIntervalSince(started) * 1000
+        for a in answers {
+            let detail =
+                a.kind == "noul"
+                ? String(format: "P(true) %.3f", a.yes)
+                : a.kind == "score" ? String(format: "score %.2f", a.score) : a.best
+            print("\(a.id): \(detail)  \(zip(a.keys, a.probabilities).map { "\($0) \(String(format: "%.3f", $1))" })")
+        }
+        print(String(format: "%@: 3 questions in %.0f ms", manager.modelName, ms))
+    }
+
+    static func main() async throws {
+        let arguments = Array(CommandLine.arguments.dropFirst())
+        guard #available(macOS 15.0, *) else { fatalError("macOS 15 required") }
+        if arguments.first == "parity", arguments.count == 3 {
+            try await parity(
+                directory: URL(fileURLWithPath: arguments[1]), fixtures: URL(fileURLWithPath: arguments[2]))
+        } else if arguments.first == "example", arguments.count == 2,
+            let model = D3Model(rawValue: "d3-\(arguments[1])-coreml")
+        {
+            try await example(model)
+        } else {
+            fputs("usage: D3Check example lite|nano|mini | parity <model dir> <fixtures dir>\n", stderr)
+            exit(2)
+        }
+    }
+}

Sources/FluidUse/ClefVision/ClefVisionTypes.swift

@@ -93,7 +93,8 @@ public enum ClefVisionError: Error, LocalizedError, Sendable {
         switch self {
         case .invalidAsset(let m): return "Invalid asset: \(m)"
         case .invalidInput(let m): return "Invalid input: \(m)"
-        case .tooLong(let tokens, let bucket): return "Record needs \(tokens) tokens; the largest bucket holds \(bucket)"
+        case .tooLong(let tokens, let bucket):
+            return "Record needs \(tokens) tokens; the largest bucket holds \(bucket)"
         case .tooManyFields(let q, let o): return "Record has \(q) questions / \(o) options; the head holds 16 / 64"
         case .checksumMismatch(let m): return "Checksum mismatch: \(m)"
         case .download(let m): return "Download failed: \(m)"
@@ -142,10 +143,13 @@ struct ClefVisionConfig: Sendable {
 
     init(json: [String: Any]) throws {
         func int(_ key: String, in dict: [String: Any]) throws -> Int {
-            guard let value = (dict[key] as? NSNumber)?.intValue else { throw ClefVisionError.invalidAsset("config.json: \(key)") }
+            guard let value = (dict[key] as? NSNumber)?.intValue else {
+                throw ClefVisionError.invalidAsset("config.json: \(key)")
+            }
             return value
         }
-        guard let tokenizer = json["tokenizer"] as? [String: Any], let embeddings = json["embeddings"] as? [String: Any],
+        guard let tokenizer = json["tokenizer"] as? [String: Any],
+            let embeddings = json["embeddings"] as? [String: Any],
             let shape = embeddings["shape"] as? [NSNumber], shape.count == 2,
             let packages = json["packages"] as? [String: Any], let lm = packages["lm"] as? [[String: Any]],
             let visionPkg = packages["vision"] as? [String: Any],
@@ -155,7 +159,11 @@ struct ClefVisionConfig: Sendable {
         else { throw ClefVisionError.invalidAsset("config.json is missing sections") }
         // one head package per LM bucket (list), or a single package (object) in older manifests
         let heads: [[String: Any]]
-        if let list = packages["head"] as? [[String: Any]] { heads = list } else if let one = packages["head"] as? [String: Any] { heads = [one] } else {
+        if let list = packages["head"] as? [[String: Any]] {
+            heads = list
+        } else if let one = packages["head"] as? [String: Any] {
+            heads = [one]
+        } else {
             throw ClefVisionError.invalidAsset("config.json: packages.head")
         }
         guard let head = heads.first else { throw ClefVisionError.invalidAsset("config.json: no head package") }
@@ -177,12 +185,40 @@ struct ClefVisionConfig: Sendable {
         let visionRope = (visionCfg["rope_parameters"] as? [String: Any])?["rope_theta"] as? NSNumber
         vision = VisionShape(
             patches: try int("patches", in: visionPkg), patchSize: try int("patch_size", in: visionCfg),
-            mergeSize: try int("spatial_merge_size", in: visionCfg), temporalPatch: try int("temporal_patch_size", in: visionCfg),
-            hidden: try int("hidden_size", in: visionCfg), headDim: try int("hidden_size", in: visionCfg) / (try int("num_heads", in: visionCfg)),
-            depthPositions: try int("num_position_embeddings", in: visionCfg), ropeTheta: visionRope?.doubleValue ?? 10_000)
+            mergeSize: try int("spatial_merge_size", in: visionCfg),
+            temporalPatch: try int("temporal_patch_size", in: visionCfg),
+            hidden: try int("hidden_size", in: visionCfg),
+            headDim: try int("hidden_size", in: visionCfg) / (try int("num_heads", in: visionCfg)),
+            depthPositions: try int("num_position_embeddings", in: visionCfg),
+            ropeTheta: visionRope?.doubleValue ?? 10_000)
         self.image = ImageProcessing(
             minPixels: try int("min_pixels", in: image), maxPixels: try int("max_pixels", in: image),
             mean: mean.map(\.floatValue), std: std.map(\.floatValue),
             rescale: (image["rescale_factor"] as? NSNumber)?.floatValue ?? 1 / 255)
     }
 }
+
+extension ClefVisionConfig {
+    /// Direct construction for other Qwen3.5 vision checkpoints (d3) that reuse the preprocessor and host math.
+    init(
+        imageTokenID: Int, visionStartID: Int, visionEndID: Int, padID: Int, vocabSize: Int, hiddenSize: Int,
+        rotaryDim: Int, ropeTheta: Double, mropeSection: [Int], lmBuckets: [Int], vision: VisionShape,
+        image: ImageProcessing
+    ) {
+        self.imageTokenID = imageTokenID
+        self.visionStartID = visionStartID
+        self.visionEndID = visionEndID
+        self.padID = padID
+        self.vocabSize = vocabSize
+        self.hiddenSize = hiddenSize
+        self.rotaryDim = rotaryDim
+        self.ropeTheta = ropeTheta
+        self.mropeSection = mropeSection
+        self.lmBuckets = lmBuckets
+        headLength = 0
+        headMaxQuestions = 0
+        headMaxOptions = 0
+        self.vision = vision
+        self.image = image
+    }
+}