From c4f807d9e30254d0a9955902996778d02f076317 Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Sun, 11 Oct 2026 14:22:26 -0400 Subject: [PATCH] feat: Decision 3.0 (d3-lite/nano/mini) Core ML runtime D3Manager runs vLLM Semantic Router's Decision 3.0 models on the GPU through Core ML: text graph per length bucket (E256/E640/E1536, fp16 compute, int8 weights), fp32 vision tower per patch bucket (P320/P1536/P3072), host-side embeddings, M-RoPE positions and the 255-way FP32 readout, one row per question. D3ModelStore downloads pinned, checksummed snapshots from the lite/, nano/ and mini/ folders of FluidInference/d3-coreml (revision 31d0a53b; all 69 pinned checksums verified against the repo). D3Check: 'example lite|nano|mini' and 'parity' against the Python Core ML host (0 token mismatches, 0 flips). Answers vs upstream PyTorch (bf16, MPS): text 2/0/1 flips of 100, images 1/24, 0/24, 0/48; ~46/65/160 ms per text question on an M5 Pro. --- Package.swift | 1 + README.md | 38 +++ Sources/D3Check/main.swift | 122 ++++++++ .../FluidUse/ClefVision/ClefVisionTypes.swift | 50 ++- Sources/FluidUse/D3/D3Manager.swift | 295 ++++++++++++++++++ Sources/FluidUse/D3/D3ModelStore.swift | 284 +++++++++++++++++ Sources/FluidUse/D3/D3Types.swift | 159 ++++++++++ Tests/FluidUseTests/D3PromptTests.swift | 45 +++ 8 files changed, 987 insertions(+), 7 deletions(-) create mode 100644 Sources/D3Check/main.swift create mode 100644 Sources/FluidUse/D3/D3Manager.swift create mode 100644 Sources/FluidUse/D3/D3ModelStore.swift create mode 100644 Sources/FluidUse/D3/D3Types.swift create mode 100644 Tests/FluidUseTests/D3PromptTests.swift diff --git a/Package.swift b/Package.swift index dd9e47f..90b883c 100644 --- a/Package.swift +++ b/Package.swift @@ -72,6 +72,7 @@ let package = Package( .executableTarget(name: "ClefCompareDemo", dependencies: ["FluidUse"], exclude: ["README.md", "demo.sh"]), .executableTarget(name: "InternDecisionCheck", dependencies: ["FluidUse"]), .executableTarget(name: "Decision2Check", dependencies: ["FluidUse"]), + .executableTarget(name: "D3Check", dependencies: ["FluidUse"]), .executableTarget( name: "IssueTriageDemo", dependencies: ["FluidUse"], exclude: ["README.md", "fetch-issues.sh"]), .executableTarget(name: "Vela2Check", dependencies: ["FluidUse"]), diff --git a/README.md b/README.md index d556357..bb34ddb 100644 --- a/README.md +++ b/README.md @@ -245,6 +245,44 @@ packed tokens) 63 / 113 ms. The upstream PyTorch runtime on the same Mac (MPS, f the Swift host against the Python reference. [Sources/IssueTriageDemo](Sources/IssueTriageDemo/README.md) labels 1,000 GitHub issues live with Kai (type, owning team, priority, flags: five decisions per issue in one call). +## Decision 3.0 + +`D3Manager` runs vLLM Semantic Router's [Decision 3.0](https://huggingface.co/collections/vllm-sr/decision-30) d3 +models (Apache-2.0, v3.1.0) on the GPU through Core ML: d3-lite 0.85B, d3-nano 2.2B and d3-mini 4.5B (Qwen3.5 hybrid +with a vision tower). A request is a state (text or JSON), optional images, and any number of named questions (choice, +yes/no, score); as upstream, each question is its own row and gets a probability for every option. Images go through +the fp32 vision tower once per request (the clef-vision preprocessor and host math), and the text graph (fp16 compute, +int8 weights) runs per question in the smallest of three length buckets. Answers vs the upstream PyTorch runtime: + +| | text (100 q) | images (24–48 q) | video (28 answers, Python host) | per text question (M5 Pro) | +|---|---|---|---|---| +| d3-lite | 2 flips | 1 flip | 0 flips | ~46 ms | +| d3-nano | 0 flips | 0 flips | 1 flip | ~65 ms | +| d3-mini | 1 flip | 0 flips | 0 flips | ~160 ms | + +The pinned snapshots download from [FluidInference/d3-coreml](https://huggingface.co/FluidInference/d3-coreml) (`lite/` +3.2 GB, `nano/` 9.1 GB, `mini/` 16 GB) on first use (macOS 15 / iOS 18). + +```swift +let model = try await D3Manager.load(from: try await D3ModelStore.ensure(.lite)) +try await model.warm() +let answers = try await model.answer( + state: .string("The order arrived damaged yesterday. The customer has a receipt and asks for a replacement today."), + images: [], // or [try await model.encode(cgImage)] + questions: [ + ("route", .choice("Which team should handle this request?", options: [ + ("returns", "Refunds, replacements and damaged deliveries"), ("billing", "Payments, invoices and charges"), + ])), + ("receipt", .noul("Does the customer have a receipt?")), + ("urgency", .score("How urgent is this request?", levels: ["Routine", "Soon", "Today"])), + ]) +print(answers[0].best, answers[1].yes, answers[2].score) +``` + +That request takes 139 ms with d3-lite on an M5 Pro. `swift run -c release D3Check example lite|nano|mini` downloads a +snapshot and answers it; `D3Check parity ` checks the Swift host against the Python +Core ML host on marked browser screenshots (0 token mismatches, 0 flips). + ## Vela 2.0 routing, safety and spans `Vela2Manager` runs vLLM Semantic Router × KR Labs' diff --git a/Sources/D3Check/main.swift b/Sources/D3Check/main.swift new file mode 100644 index 0000000..ffb585e --- /dev/null +++ b/Sources/D3Check/main.swift @@ -0,0 +1,122 @@ +import CoreGraphics +import FluidUse +import Foundation +import ImageIO + +/// d3 checks. +/// +/// swift run -c release D3Check example lite|nano|mini (downloads the pinned snapshot, answers the card request) +/// swift run -c release D3Check parity (marked screenshots vs the Python Core ML host) +/// +/// Fixtures: model-lab models/d3-demos/browser/make_swift_fixtures.py (token ids + probabilities per screenshot). +@main +struct D3Check { + struct VisionFixtures: Decodable { let fixtures: [VisionFixture] } + + struct VisionFixture: Decodable { + struct Option: Decodable { + let key: String + let description: String + } + let image: String + let state: String + let instructions: String + let options: [Option] + let ids: [Int] + let probabilities: [Double] + } + + @available(macOS 15.0, *) + static func parity(directory: URL, fixtures folder: URL) async throws { + let fixtures = try JSONDecoder().decode( + VisionFixtures.self, from: Data(contentsOf: folder.appendingPathComponent("fixtures.json")) + ).fixtures + let started = Date() + let manager = try await D3Manager.load(from: directory) + print(String(format: "loaded in %.1f s", Date().timeIntervalSince(started))) + var worst = 0.0 + var mismatches = 0 + var flips = 0 + for (index, fixture) in fixtures.enumerated() { + guard let source = CGImageSourceCreateWithURL(folder.appendingPathComponent(fixture.image) as CFURL, nil), + let image = CGImageSourceCreateImageAtIndex(source, 0, nil) + else { throw D3Error.invalidAsset(fixture.image) } + let question = D3Question.choice( + fixture.instructions, options: fixture.options.map { ($0.key, $0.description) }) + for pass in 0..<(index == 0 ? 2 : 1) { // the first call per bucket also pays the GPU setup + let encoded = try await manager.encode(image) + let ids = try await manager.tokens(state: .string(fixture.state), question: question, images: [encoded]) + let answer = try await manager.answer( + state: .string(fixture.state), images: [encoded], questions: [("click", question)])[0] + guard pass == (index == 0 ? 1 : 0) else { continue } + if ids != fixture.ids { mismatches += 1 } + let d = zip(answer.probabilities, fixture.probabilities).map { abs($0 - $1) }.max() ?? 0 + worst = max(worst, d) + let theirs = fixture.probabilities.indices.max { fixture.probabilities[$0] < fixture.probabilities[$1] } + let ours = answer.probabilities.indices.max { answer.probabilities[$0] < answer.probabilities[$1] } + if theirs != ours { flips += 1 } + print( + String( + format: "%@: %d tokens (%@), choice %@, max|dp| %.4f, vision %.0f ms + text %.0f ms", + fixture.image, ids.count, + ids == fixture.ids ? "ids match" : "IDS DIFFER (\(fixture.ids.count))", + answer.best, d, encoded.milliseconds, answer.milliseconds)) + } + } + print( + "\(fixtures.count) screenshots: token mismatches \(mismatches), flips \(flips), max|dp| \(String(format: "%.4f", worst))" + ) + if mismatches > 0 || flips > 0 { exit(1) } + } + + @available(macOS 15.0, *) + static func example(_ model: D3Model) async throws { + let directory = try await D3ModelStore.ensure(model) { file, bytes in + if bytes > 0 { print("downloaded \(file) (\(bytes / 1_000_000) MB)") } + } + let manager = try await D3Manager.load(from: directory) + let state = OrderedJSON.string( + "The order arrived damaged yesterday. The customer has a receipt and asks for a replacement today.") + let questions: [(id: String, question: D3Question)] = [ + ( + "route", + .choice( + "Which team should handle this request?", + options: [ + ("returns", "Refunds, replacements and damaged deliveries"), + ("billing", "Payments, invoices and charges"), ("technical", "Product setup and faults"), + ]) + ), + ("receipt", .noul("Does the customer have a receipt?")), + ("urgency", .score("How urgent is this request?", levels: ["Routine", "Soon", "Today"])), + ] + _ = try await manager.answer(state: state, images: [], questions: questions) // first call pays the setup + let started = Date() + let answers = try await manager.answer(state: state, images: [], questions: questions) + let ms = Date().timeIntervalSince(started) * 1000 + for a in answers { + let detail = + a.kind == "noul" + ? String(format: "P(true) %.3f", a.yes) + : a.kind == "score" ? String(format: "score %.2f", a.score) : a.best + print("\(a.id): \(detail) \(zip(a.keys, a.probabilities).map { "\($0) \(String(format: "%.3f", $1))" })") + } + print(String(format: "%@: 3 questions in %.0f ms", manager.modelName, ms)) + } + + static func main() async throws { + let arguments = Array(CommandLine.arguments.dropFirst()) + guard #available(macOS 15.0, *) else { fatalError("macOS 15 required") } + if arguments.first == "parity", arguments.count == 3 { + try await parity( + directory: URL(fileURLWithPath: arguments[1]), fixtures: URL(fileURLWithPath: arguments[2])) + } else if arguments.first == "example", arguments.count == 2, + let model = D3Model(rawValue: "d3-\(arguments[1])-coreml") + { + try await example(model) + } else { + fputs("usage: D3Check example lite|nano|mini | parity \n", stderr) + exit(2) + } + } +} diff --git a/Sources/FluidUse/ClefVision/ClefVisionTypes.swift b/Sources/FluidUse/ClefVision/ClefVisionTypes.swift index d26e671..c9e9d4f 100644 --- a/Sources/FluidUse/ClefVision/ClefVisionTypes.swift +++ b/Sources/FluidUse/ClefVision/ClefVisionTypes.swift @@ -93,7 +93,8 @@ public enum ClefVisionError: Error, LocalizedError, Sendable { switch self { case .invalidAsset(let m): return "Invalid asset: \(m)" case .invalidInput(let m): return "Invalid input: \(m)" - case .tooLong(let tokens, let bucket): return "Record needs \(tokens) tokens; the largest bucket holds \(bucket)" + case .tooLong(let tokens, let bucket): + return "Record needs \(tokens) tokens; the largest bucket holds \(bucket)" case .tooManyFields(let q, let o): return "Record has \(q) questions / \(o) options; the head holds 16 / 64" case .checksumMismatch(let m): return "Checksum mismatch: \(m)" case .download(let m): return "Download failed: \(m)" @@ -142,10 +143,13 @@ struct ClefVisionConfig: Sendable { init(json: [String: Any]) throws { func int(_ key: String, in dict: [String: Any]) throws -> Int { - guard let value = (dict[key] as? NSNumber)?.intValue else { throw ClefVisionError.invalidAsset("config.json: \(key)") } + guard let value = (dict[key] as? NSNumber)?.intValue else { + throw ClefVisionError.invalidAsset("config.json: \(key)") + } return value } - guard let tokenizer = json["tokenizer"] as? [String: Any], let embeddings = json["embeddings"] as? [String: Any], + guard let tokenizer = json["tokenizer"] as? [String: Any], + let embeddings = json["embeddings"] as? [String: Any], let shape = embeddings["shape"] as? [NSNumber], shape.count == 2, let packages = json["packages"] as? [String: Any], let lm = packages["lm"] as? [[String: Any]], let visionPkg = packages["vision"] as? [String: Any], @@ -155,7 +159,11 @@ struct ClefVisionConfig: Sendable { else { throw ClefVisionError.invalidAsset("config.json is missing sections") } // one head package per LM bucket (list), or a single package (object) in older manifests let heads: [[String: Any]] - if let list = packages["head"] as? [[String: Any]] { heads = list } else if let one = packages["head"] as? [String: Any] { heads = [one] } else { + if let list = packages["head"] as? [[String: Any]] { + heads = list + } else if let one = packages["head"] as? [String: Any] { + heads = [one] + } else { throw ClefVisionError.invalidAsset("config.json: packages.head") } guard let head = heads.first else { throw ClefVisionError.invalidAsset("config.json: no head package") } @@ -177,12 +185,40 @@ struct ClefVisionConfig: Sendable { let visionRope = (visionCfg["rope_parameters"] as? [String: Any])?["rope_theta"] as? NSNumber vision = VisionShape( patches: try int("patches", in: visionPkg), patchSize: try int("patch_size", in: visionCfg), - mergeSize: try int("spatial_merge_size", in: visionCfg), temporalPatch: try int("temporal_patch_size", in: visionCfg), - hidden: try int("hidden_size", in: visionCfg), headDim: try int("hidden_size", in: visionCfg) / (try int("num_heads", in: visionCfg)), - depthPositions: try int("num_position_embeddings", in: visionCfg), ropeTheta: visionRope?.doubleValue ?? 10_000) + mergeSize: try int("spatial_merge_size", in: visionCfg), + temporalPatch: try int("temporal_patch_size", in: visionCfg), + hidden: try int("hidden_size", in: visionCfg), + headDim: try int("hidden_size", in: visionCfg) / (try int("num_heads", in: visionCfg)), + depthPositions: try int("num_position_embeddings", in: visionCfg), + ropeTheta: visionRope?.doubleValue ?? 10_000) self.image = ImageProcessing( minPixels: try int("min_pixels", in: image), maxPixels: try int("max_pixels", in: image), mean: mean.map(\.floatValue), std: std.map(\.floatValue), rescale: (image["rescale_factor"] as? NSNumber)?.floatValue ?? 1 / 255) } } + +extension ClefVisionConfig { + /// Direct construction for other Qwen3.5 vision checkpoints (d3) that reuse the preprocessor and host math. + init( + imageTokenID: Int, visionStartID: Int, visionEndID: Int, padID: Int, vocabSize: Int, hiddenSize: Int, + rotaryDim: Int, ropeTheta: Double, mropeSection: [Int], lmBuckets: [Int], vision: VisionShape, + image: ImageProcessing + ) { + self.imageTokenID = imageTokenID + self.visionStartID = visionStartID + self.visionEndID = visionEndID + self.padID = padID + self.vocabSize = vocabSize + self.hiddenSize = hiddenSize + self.rotaryDim = rotaryDim + self.ropeTheta = ropeTheta + self.mropeSection = mropeSection + self.lmBuckets = lmBuckets + headLength = 0 + headMaxQuestions = 0 + headMaxOptions = 0 + self.vision = vision + self.image = image + } +} diff --git a/Sources/FluidUse/D3/D3Manager.swift b/Sources/FluidUse/D3/D3Manager.swift new file mode 100644 index 0000000..4651db7 --- /dev/null +++ b/Sources/FluidUse/D3/D3Manager.swift @@ -0,0 +1,295 @@ +@preconcurrency import CoreML +import CoreGraphics +import Foundation + +/// vLLM Semantic Router Decision 3.0 (d3-lite / d3-nano / d3-mini) on Core ML, text and image requests: the request's images go through the fp32 vision tower once, and +/// every question's row (system prompt, image placeholders, state, question, options) runs the fp16 text graph with +/// the image tokens spliced in at the placeholder positions and 3-D M-RoPE positions. FP32 readout on the host. +/// +/// Image preprocessing, the vision tower's inputs and the M-RoPE positions are clef-vision's host code +/// (`ClefImagePreprocessor`, `ClefVisionHost`): same Qwen3.5 vision family. +/// +/// The directory holds `coreml_config.json`, `tokenizer.json`, `readout.f32`, `embeddings.f16` ([V, H] fp16), +/// `pos_embed.f32`, `vision_P{patches}.mlpackage` and `d3_E{tokens}.mlpackage` (or their compiled `.mlmodelc`). +@available(macOS 15.0, iOS 18.0, *) +public actor D3Manager { + private struct TextBucket { + let length: Int + let model: MLModel + let hidden: MLMultiArray + let cos: MLMultiArray + let sin: MLMultiArray + let keyValid: MLMultiArray + let lastOnehot: MLMultiArray + let features: MLDictionaryFeatureProvider + } + + private struct VisionBucket { + let patches: Int + let model: MLModel + let config: ClefVisionConfig + } + + public nonisolated let modelName: String + private let tokenizer: QwenBPETokenizer + private let codes: [String] + private let readout: [Float] + private let embeddings: Data + private let posEmbed: [Float] + private let hidden: Int + private let temperature: Double + private let padID: Int + private let imageTokenID: Int + private let mergeSize: Int + private let rotaryDim: Int + private let ropeTheta: Double + private let mropeSection: [Int] + private let vision: [VisionBucket] + private let text: [TextBucket] + + private init( + modelName: String, tokenizer: QwenBPETokenizer, codes: [String], readout: [Float], embeddings: Data, + posEmbed: [Float], hidden: Int, temperature: Double, padID: Int, imageTokenID: Int, mergeSize: Int, + rotaryDim: Int, ropeTheta: Double, mropeSection: [Int], vision: [VisionBucket], text: [TextBucket] + ) { + self.modelName = modelName + self.tokenizer = tokenizer + self.codes = codes + self.readout = readout + self.embeddings = embeddings + self.posEmbed = posEmbed + self.hidden = hidden + self.temperature = temperature + self.padID = padID + self.imageTokenID = imageTokenID + self.mergeSize = mergeSize + self.rotaryDim = rotaryDim + self.ropeTheta = ropeTheta + self.mropeSection = mropeSection + self.vision = vision + self.text = text + } + + public static func load( + from directory: URL, computeUnits: MLComputeUnits = .cpuAndGPU + ) async throws + -> D3Manager + { + func file(_ name: String) -> URL { directory.appendingPathComponent(name) } + guard + let config = try JSONSerialization.jsonObject(with: Data(contentsOf: file("coreml_config.json"))) + as? [String: Any], + let modelName = config["model_name"] as? String, let codes = config["codes"] as? [String], + codes.count == D3Prompt.maxOptions, let pad = config["pad_token_id"] as? Int, + let imageToken = config["image_token_id"] as? Int, let start = config["vision_start_token_id"] as? Int, + let end = config["vision_end_token_id"] as? Int, let hidden = config["hidden_size"] as? Int, + let vocab = config["vocab_size"] as? Int, let rope = config["rope"] as? [String: Any], + let rotary = rope["rotary_dim"] as? Int, let theta = (rope["theta"] as? NSNumber)?.doubleValue, + let section = rope["mrope_section"] as? [Int], let v = config["vision"] as? [String: Any], + let image = config["image"] as? [String: Any], let visionBuckets = config["vision_buckets"] as? [Int], + let textBuckets = config["text_buckets"] as? [Int] + else { throw D3Error.invalidAsset("coreml_config.json is missing fields") } + func int(_ key: String) throws -> Int { + guard let value = (v[key] as? NSNumber)?.intValue else { throw D3Error.invalidAsset("vision.\(key)") } + return value + } + let merge = try int("merge_size") + let processing = ClefVisionConfig.ImageProcessing( + minPixels: (image["min_pixels"] as? Int) ?? 65_536, maxPixels: (image["max_pixels"] as? Int) ?? 1_638_400, + mean: ((image["mean"] as? [NSNumber]) ?? [0.5, 0.5, 0.5]).map(\.floatValue), + std: ((image["std"] as? [NSNumber]) ?? [0.5, 0.5, 0.5]).map(\.floatValue), + rescale: (image["rescale"] as? NSNumber)?.floatValue ?? 1 / 255) + let readout = try Data(contentsOf: file("readout.f32")).withUnsafeBytes { Array($0.bindMemory(to: Float.self)) } + let posEmbed = try Data(contentsOf: file("pos_embed.f32")).withUnsafeBytes { + Array($0.bindMemory(to: Float.self)) + } + let embeddings = try Data(contentsOf: file("embeddings.f16"), options: .alwaysMapped) + guard embeddings.count == vocab * hidden * 2 else { throw D3Error.invalidAsset("embeddings.f16 size") } + + func compiled(_ name: String) async throws -> URL { + let c = file(name + ".mlmodelc") + if FileManager.default.fileExists(atPath: c.path) { return c } + // The release may symlink packages; compile next to the link, inside the release. + let temporary = try await MLModel.compileModel(at: file(name + ".mlpackage").resolvingSymlinksInPath()) + try? FileManager.default.moveItem(at: temporary, to: c) + return FileManager.default.fileExists(atPath: c.path) ? c : temporary + } + let configuration = MLModelConfiguration() + configuration.computeUnits = computeUnits + var visions: [VisionBucket] = [] + for patches in visionBuckets.sorted() { + let shape = ClefVisionConfig.VisionShape( + patches: patches, patchSize: try int("patch_size"), mergeSize: merge, + temporalPatch: try int("temporal_patch_size"), hidden: try int("hidden_size"), + headDim: try int("hidden_size") / (try int("num_heads")), + depthPositions: try int("num_position_embeddings"), + ropeTheta: (v["rope_theta"] as? NSNumber)?.doubleValue ?? 10_000) + let clef = ClefVisionConfig( + imageTokenID: imageToken, visionStartID: start, visionEndID: end, padID: pad, vocabSize: vocab, + hiddenSize: hidden, rotaryDim: rotary, ropeTheta: theta, mropeSection: section, lmBuckets: textBuckets, + vision: shape, image: processing) + let model = try await MLModel.load( + contentsOf: try await compiled("vision_P\(patches)"), configuration: configuration) + visions.append(VisionBucket(patches: patches, model: model, config: clef)) + } + var texts: [TextBucket] = [] + for length in textBuckets.sorted() { + let model = try await MLModel.load( + contentsOf: try await compiled("d3_E\(length)"), configuration: configuration) + let l = NSNumber(value: length) + let hiddenArray = try MLMultiArray(shape: [1, l, NSNumber(value: hidden)], dataType: .float16) + let cos = try MLMultiArray(shape: [l, NSNumber(value: rotary)], dataType: .float16) + let sin = try MLMultiArray(shape: [l, NSNumber(value: rotary)], dataType: .float16) + let keyValid = try MLMultiArray(shape: [l], dataType: .float16) + let lastOnehot = try MLMultiArray(shape: [1, l], dataType: .float16) + let features = try MLDictionaryFeatureProvider(dictionary: [ + "hidden": hiddenArray, "cos": cos, "sin": sin, "key_valid": keyValid, "last_onehot": lastOnehot, + ]) + texts.append( + TextBucket( + length: length, model: model, hidden: hiddenArray, cos: cos, sin: sin, keyValid: keyValid, + lastOnehot: lastOnehot, features: features)) + } + guard !visions.isEmpty, !texts.isEmpty else { throw D3Error.invalidAsset("no buckets") } + return D3Manager( + modelName: modelName, + tokenizer: try QwenBPETokenizer(tokenizerJsonURL: file("tokenizer.json")), codes: codes, readout: readout, + embeddings: embeddings, posEmbed: posEmbed, hidden: hidden, + temperature: (config["temperature"] as? NSNumber)?.doubleValue ?? 1, padID: pad, imageTokenID: imageToken, + mergeSize: merge, rotaryDim: rotary, ropeTheta: theta, mropeSection: section, vision: visions, text: texts) + } + + /// One encoded image: its merged tokens [count × hidden] and grid (in patches). + public struct EncodedImage: Sendable { + let tokens: [Float] + let gridH: Int + let gridW: Int + public let milliseconds: Double + public var count: Int { gridH * gridW / 4 } + } + + /// The vision tower for one image (reuse the result for every question about it). + public func encode(_ image: CGImage) throws -> EncodedImage { + let first = vision[0] + let patches = try ClefImagePreprocessor.patches(from: image, config: first.config) + guard let bucket = vision.first(where: { $0.patches >= patches.count }) else { + throw D3Error.tooLong(needed: patches.count, budget: vision.last?.patches ?? 0) + } + let inputs = try posEmbed.withUnsafeBufferPointer { + try ClefVisionHost.visionInputs(patches, posEmbed: $0, config: bucket.config) + } + let started = DispatchTime.now().uptimeNanoseconds + let output = try bucket.model.prediction(from: inputs.features) + let milliseconds = Double(DispatchTime.now().uptimeNanoseconds - started) / 1e6 + guard let array = output.featureValue(for: "tokens")?.multiArrayValue else { + throw D3Error.invalidOutput("vision tokens") + } + let count = inputs.mergedTokens * hidden + let pointer = array.dataPointer.assumingMemoryBound(to: Float.self) + return EncodedImage( + tokens: Array(UnsafeBufferPointer(start: pointer, count: count)), gridH: patches.gridH, + gridW: patches.gridW, milliseconds: milliseconds) + } + + /// The token ids of one question's row about `images` (for parity checks). + public func tokens(state: OrderedJSON, question: D3Question, images: [EncodedImage]) throws -> [Int] { + try tokenizer.encode( + try D3Prompt.render(state: state, question: question, codes: codes, imageTokens: images.map(\.count))) + } + + /// Answers each question from its own row, every row seeing all `images` (none for a text request). + public func answer( + state: OrderedJSON, images: [EncodedImage], questions: [(id: String, question: D3Question)] + ) + throws -> [D3Answer] + { + try questions.map { item in + let ids = try tokens(state: state, question: item.question, images: images) + let started = DispatchTime.now().uptimeNanoseconds + let final = try run(ids: ids, images: images) + let milliseconds = Double(DispatchTime.now().uptimeNanoseconds - started) / 1e6 + let count = item.question.keys.count + var logits = (0.. Double in + var sum = 0.0 + for k in 0..