Portrait mode: a third capture mode with the background blurred from depth

RemoteCam/CameraProgressOverlayView.swift

@@ -86,6 +86,7 @@ struct CameraProgressOverlayView: View {
         case .Photo: return "Photo"
         case .Video: return "Video"
         case .Shorts: return "Shorts"
+        case .Portrait: return "Portrait"
         }
     }
 }

RemoteCam/CameraScreenView.swift

@@ -331,6 +331,7 @@ struct CameraStandbyView: View {
         case .Photo: return NSLocalizedString("Photo", comment: "capture mode")
         case .Video: return NSLocalizedString("Video", comment: "capture mode")
         case .Shorts: return NSLocalizedString("Shorts", comment: "capture mode")
+        case .Portrait: return NSLocalizedString("Portrait", comment: "capture mode")
         }
     }
 }

RemoteCam/CameraViewModel.swift

@@ -88,7 +88,7 @@ class CameraViewModel: ObservableObject {
             switch mode {
             case .Video, .Shorts:
                 self.qualityInfo = "\(resolution.displayName) \(frameRate.displayName)fps"
-            case .Photo:
+            case .Photo, .Portrait:
                 var info = photoFormat.displayName
                 if hdrMode == .on { info += " HDR" }
                 self.qualityInfo = info

RemoteCam/CaptureEngine.swift

@@ -8,6 +8,7 @@
 import Atomics
 import UIKit
 import AVFoundation
+import CoreImage
 
 /// Classifies capture-session runtime errors by recency. One error is
 /// transient (media services reset — resume and move on); a repeat inside the
@@ -196,7 +197,7 @@ final class CaptureEngine: NSObject, AVCapturePhotoCaptureDelegate {
         }
         self.captureSession.beginConfiguration()
         self.captureSession.sessionPreset =
-            (self.captureMode == .Photo && Self.photoPresetSupported
+            (self.captureMode.isStill && Self.photoPresetSupported
              && self.captureSession.canSetSessionPreset(.photo))
             ? .photo : .high
 
@@ -234,6 +235,7 @@ final class CaptureEngine: NSObject, AVCapturePhotoCaptureDelegate {
             self.configSessionOutput()
             self.captureSession.commitConfiguration()
             self.reconcilePreviewOutputLocked()
+            self.applyDepthDeliveryLocked()
             self.updateMaxPhotoDimensionsLocked()
             self.currentPhotoFormat = Self.defaultPhotoFormat(
                 hevcAvailable: self.photoOutput.availablePhotoCodecTypes.contains(.hevc))
@@ -409,6 +411,7 @@ final class CaptureEngine: NSObject, AVCapturePhotoCaptureDelegate {
         rotateOutputsLocked(orientation: orientation)
         let newFlashMode: AVCaptureDevice.FlashMode? = newDevice.hasFlash ? cameraSettings.flashMode : nil
         captureSession.commitConfiguration()
+        applyDepthDeliveryLocked()   // the new device may or may not deliver depth
         reconcilePreviewOutputLocked()
         updateMaxPhotoDimensionsLocked()
         applyDesiredTorchLocked()   // restore torch onto the new camera (no-op if it has none)
@@ -1105,6 +1108,7 @@ final class CaptureEngine: NSObject, AVCapturePhotoCaptureDelegate {
             aspectRatio: currentAspectRatio,
             exposure: exposureStateLocked(currentDevice),
             cameraMode: captureMode,
+            supportsPortrait: Self.supportsPortrait(currentDevice),
             error: nil
         )
 
@@ -1779,30 +1783,64 @@ final class CaptureEngine: NSObject, AVCapturePhotoCaptureDelegate {
 
     /// Follows the rig's `currentCameraMode`. Stores the mode even before the
     /// session is set up (setup reads it); once live, reconfigures the session:
-    /// Photo gets the full-sensor `.photo` preset, Video/Shorts get the selected
-    /// video format back. Dedupes, so repeated sets are free.
+    /// Photo and Portrait get the full-sensor `.photo` preset (Portrait adds
+    /// depth delivery), Video/Shorts get the selected video format back.
+    /// Dedupes, so repeated sets are free.
     func setCaptureMode(_ mode: RecordingMode) {
         sessionQueue.async {
             guard mode != self.captureMode else { return }
+            let previous = self.captureMode
             self.captureMode = mode
             guard self.videoDeviceInput?.device != nil else { return }
-            self.applyCaptureModeLocked()
+            self.applyCaptureModeLocked(from: previous)
         }
     }
 
     /// Reconfigures the session for the stored mode. A real reconfiguration
     /// changes the frame dimensions (16:9 video ↔ 4:3 full sensor) — a scene
-    /// cut for the preview encoder, announced like a device swap.
-    private func applyCaptureModeLocked() {
+    /// cut for the preview encoder, announced like a device swap. Photo ↔
+    /// Portrait keeps the photo format and only toggles depth delivery.
+    private func applyCaptureModeLocked(from previous: RecordingMode) {
         dispatchPrecondition(condition: .onQueue(sessionQueue))
-        let changed = captureMode == .Photo
-            ? applyPhotoFormatLocked()
-            : setVideoQualityLocked(resolution: currentVideoResolution,
-                                    frameRate: currentVideoFrameRate,
-                                    isRecording: false) != nil
+        let changed: Bool
+        if captureMode.isStill && previous.isStill {
+            changed = false
+        } else if captureMode.isStill {
+            changed = applyPhotoFormatLocked()
+        } else {
+            changed = setVideoQualityLocked(resolution: currentVideoResolution,
+                                            frameRate: currentVideoFrameRate,
+                                            isRecording: false) != nil
+        }
+        applyDepthDeliveryLocked()
         if changed { onDeviceSwapped?() }
     }
 
+    /// Whether `device` can take Portrait stills: it has formats that deliver
+    /// depth (dual/triple back cameras, TrueDepth front) and the platform runs
+    /// the photo preset (Mac cameras have no depth).
+    static func supportsPortrait(_ device: AVCaptureDevice) -> Bool {
+        photoPresetSupported && device.formats.contains { !$0.supportedDepthDataFormats.isEmpty }
+    }
+
+    /// Depth (and the person matte, where offered) is delivered only in
+    /// Portrait. Turning it on can narrow a virtual device's zoom range and
+    /// photo sizes, so the format tail re-clamps zoom and the photo ceiling.
+    private func applyDepthDeliveryLocked() {
+        dispatchPrecondition(condition: .onQueue(sessionQueue))
+        let depth = captureMode == .Portrait && photoOutput.isDepthDataDeliverySupported
+        let matte = depth && photoOutput.isPortraitEffectsMatteDeliverySupported
+        guard photoOutput.isDepthDataDeliveryEnabled != depth
+                || photoOutput.isPortraitEffectsMatteDeliveryEnabled != matte else { return }
+        captureSession.beginConfiguration()
+        if !matte { photoOutput.isPortraitEffectsMatteDeliveryEnabled = false }
+        photoOutput.isDepthDataDeliveryEnabled = depth
+        if matte { photoOutput.isPortraitEffectsMatteDeliveryEnabled = true }
+        captureSession.commitConfiguration()
+        restoreAfterFormatChangeLocked()
+        onStatusChanged?()
+    }
+
     /// Puts the session on the full-sensor `.photo` preset, keeping the
     /// device's frame rate (`resolveFrameRate` clamps it into the photo
     /// format's own ranges). Where the preset is unavailable the session
@@ -1987,16 +2025,22 @@ final class CaptureEngine: NSObject, AVCapturePhotoCaptureDelegate {
         hevcAvailable ? .heif : .jpeg
     }
 
+    /// Portrait stills cap at 12MP. Larger sizes with depth on are untested,
+    /// and an unsupported photo size raises an uncatchable exception.
+    static let portraitMaxPhotoPixels: Int64 = 12_600_000
+
     /// Single point that keeps the photo output asking for everything the
     /// ACTIVE format can produce (12/24/48MP on modern iPhones). Must run
     /// after the configuration commit — `activeFormat` settles there.
     private func updateMaxPhotoDimensionsLocked() {
         dispatchPrecondition(condition: .onQueue(sessionQueue))
         guard let device = videoDeviceInput?.device else { return }
         if #available(iOS 16.0, macCatalyst 16.0, *) {
-            guard let dims = device.activeFormat.supportedMaxPhotoDimensions
-                .max(by: { Int64($0.width) * Int64($0.height) < Int64($1.width) * Int64($1.height) })
-            else { return }
+            let pixels = { (d: CMVideoDimensions) in Int64(d.width) * Int64(d.height) }
+            let candidates = device.activeFormat.supportedMaxPhotoDimensions.filter {
+                !photoOutput.isDepthDataDeliveryEnabled || pixels($0) <= Self.portraitMaxPhotoPixels
+            }
+            guard let dims = candidates.max(by: { pixels($0) < pixels($1) }) else { return }
             photoOutput.maxPhotoDimensions = dims
         }
     }
@@ -2013,7 +2057,7 @@ final class CaptureEngine: NSObject, AVCapturePhotoCaptureDelegate {
         guard !isRecording else { return nil }
         guard let device = videoDeviceInput?.device else { return nil }
 
-        guard captureMode != .Photo else {
+        guard !captureMode.isStill else {
             currentVideoResolution = resolution
             currentVideoFrameRate = frameRate
             fpsSetting.store(frameRate.value, ordering: .relaxed)
@@ -2124,15 +2168,54 @@ final class CaptureEngine: NSObject, AVCapturePhotoCaptureDelegate {
             return nil
         }
 
+        return encode(croppedCG, like: source, type: type)
+    }
+
+    /// Simulated aperture for Portrait stills; Apple's Camera opens at f/4.5.
+    static let portraitAperture: Float = 4.5
+    private static let portraitContext = CIContext()
+
+    /// A Portrait still: Core Image's portrait filter blurs the background
+    /// from the photo's own depth (and person matte, when delivered), then the
+    /// result is cropped to the aspect ratio and encoded like `cropPhotoData`
+    /// (sensor space, EXIF orientation and metadata carried). The blur is baked
+    /// in; the depth is not kept. Nil when the data carries no depth, so the
+    /// caller falls back to a plain crop.
+    static func portraitPhotoData(_ data: Data, to aspectRatio: AspectRatio) -> Data? {
+        guard let source = CGImageSourceCreateWithData(data as CFData, nil),
+              let type = CGImageSourceGetType(source),
+              let filter = portraitContext.depthBlurEffectFilter(forImageData: data, options: nil) else {
+            debugLog("portraitPhotoData: no depth in photo data (\(data.count) bytes)")
+            return nil
+        }
+        filter.setValue(portraitAperture, forKey: "inputAperture")
+        guard let blurred = filter.outputImage else { return nil }
+        let extent = blurred.extent
+        let rect = cropRect(sourceWidth: extent.width, sourceHeight: extent.height,
+                            aspectRatio: aspectRatio)?
+            .offsetBy(dx: extent.minX, dy: extent.minY) ?? extent
+        let colorSpace = CGImageSourceCreateImageAtIndex(source, 0, nil)?.colorSpace
+            ?? CGColorSpace(name: CGColorSpace.sRGB)!
+        guard let cgImage = portraitContext.createCGImage(
+            blurred, from: rect, format: .RGBA8, colorSpace: colorSpace) else {
+            debugLog("portraitPhotoData: render failed for rect \(rect)")
+            return nil
+        }
+        return encode(cgImage, like: source, type: type)
+    }
+
+    /// Encodes `image` into `type` with `source`'s metadata, EXIF pixel
+    /// dimensions rewritten to the image's size.
+    private static func encode(_ image: CGImage, like source: CGImageSource, type: CFString) -> Data? {
         let properties = destinationProperties(
-            of: source, croppedWidth: croppedCG.width, croppedHeight: croppedCG.height)
+            of: source, croppedWidth: image.width, croppedHeight: image.height)
         let output = NSMutableData()
         guard let dest = CGImageDestinationCreateWithData(output, type, 1, nil) else {
             return nil
         }
-        CGImageDestinationAddImage(dest, croppedCG, properties as CFDictionary)
+        CGImageDestinationAddImage(dest, image, properties as CFDictionary)
         guard CGImageDestinationFinalize(dest) else {
-            debugLog("cropPhotoData: CGImageDestinationFinalize failed")
+            debugLog("encode: CGImageDestinationFinalize failed")
             return nil
         }
         return output as Data
@@ -2187,6 +2270,10 @@ final class CaptureEngine: NSObject, AVCapturePhotoCaptureDelegate {
             newSettings.isHighResolutionPhotoEnabled = settings.isHighResolutionPhotoEnabled
         }
         newSettings.photoQualityPrioritization = .quality
+        if photoOutput.isDepthDataDeliveryEnabled {
+            newSettings.isDepthDataDeliveryEnabled = true
+            newSettings.isPortraitEffectsMatteDeliveryEnabled = photoOutput.isPortraitEffectsMatteDeliveryEnabled
+        }
         return newSettings
     }
 
@@ -2213,6 +2300,10 @@ final class CaptureEngine: NSObject, AVCapturePhotoCaptureDelegate {
         // through the session queue. Any ratio other than the still's own is
         // a crop of the full capture; a failed crop delivers the full frame.
         let aspectRatio = currentAspectRatioValue()
+        if photo.depthData != nil, let portrait = Self.portraitPhotoData(photoData, to: aspectRatio) {
+            onPicture?(portrait, nil)
+            return
+        }
         onPicture?(Self.cropPhotoData(photoData, to: aspectRatio) ?? photoData, nil)
     }
 }

RemoteCam/FlatBufferSchemas.fbs

@@ -189,7 +189,9 @@ enum RecordingModeEnum : byte {
     Unknown = 0,
     Photo = 1,
     Video = 2,
-    Shorts = 3
+    Shorts = 3,
+    Portrait = 4   // a still with the background blurred from depth. Only
+                   // sent to a camera whose CameraState sets supports_portrait.
 }
 
 enum AspectRatioEnum : byte {
@@ -391,6 +393,10 @@ table CameraState {
     // stills, video = the selected video format). Absent/Unknown = peer
     // predates mode pre-warming; a director must not send SetCameraMode.
     camera_mode: RecordingModeEnum;
+    // The active camera can take Portrait stills (it delivers depth). Moves
+    // with every device swap. False/absent = a director must not send
+    // Portrait; it offers the mode only when every camera in the rig sets it.
+    supports_portrait: bool;
 }
 
 table CameraCapabilities {

RemoteCam/FlatBufferSchemas_generated.swift

@@ -227,8 +227,9 @@ public enum RemoteShutter_RecordingModeEnum: Int8, Enum, Verifiable {
   case photo = 1
   case video = 2
   case shorts = 3
+  case portrait = 4
 
-  public static var max: RemoteShutter_RecordingModeEnum { return .shorts }
+  public static var max: RemoteShutter_RecordingModeEnum { return .portrait }
   public static var min: RemoteShutter_RecordingModeEnum { return .unknown }
 }
 
@@ -1160,6 +1161,7 @@ public struct RemoteShutter_CameraState: FlatBufferObject, Verifiable {
     case previewMode = 26
     case exposure = 28
     case cameraMode = 30
+    case supportsPortrait = 32
     var v: Int32 { Int32(self.rawValue) }
     var p: VOffset { self.rawValue }
   }
@@ -1179,7 +1181,8 @@ public struct RemoteShutter_CameraState: FlatBufferObject, Verifiable {
   public var previewMode: RemoteShutter_CameraPreviewModeEnum { let o = _accessor.offset(VTOFFSET.previewMode.v); return o == 0 ? .unknown : RemoteShutter_CameraPreviewModeEnum(rawValue: _accessor.readBuffer(of: Int8.self, at: o)) ?? .unknown }
   public var exposure: RemoteShutter_ExposureState? { let o = _accessor.offset(VTOFFSET.exposure.v); return o == 0 ? nil : RemoteShutter_ExposureState(_accessor.bb, o: _accessor.indirect(o + _accessor.position)) }
   public var cameraMode: RemoteShutter_RecordingModeEnum { let o = _accessor.offset(VTOFFSET.cameraMode.v); return o == 0 ? .unknown : RemoteShutter_RecordingModeEnum(rawValue: _accessor.readBuffer(of: Int8.self, at: o)) ?? .unknown }
-  public static func startCameraState(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 14) }
+  public var supportsPortrait: Bool { let o = _accessor.offset(VTOFFSET.supportsPortrait.v); return o == 0 ? false : _accessor.readBuffer(of: Bool.self, at: o) }
+  public static func startCameraState(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 15) }
   public static func add(currentCamera: RemoteShutter_CameraPosition, _ fbb: inout FlatBufferBuilder) { fbb.add(element: currentCamera.rawValue, def: 0, at: VTOFFSET.currentCamera.p) }
   public static func add(currentLens: RemoteShutter_CameraLensType, _ fbb: inout FlatBufferBuilder) { fbb.add(element: currentLens.rawValue, def: 0, at: VTOFFSET.currentLens.p) }
   public static func add(zoomFactor: Double, _ fbb: inout FlatBufferBuilder) { fbb.add(element: zoomFactor, def: 0.0, at: VTOFFSET.zoomFactor.p) }
@@ -1194,6 +1197,8 @@ public struct RemoteShutter_CameraState: FlatBufferObject, Verifiable {
   public static func add(previewMode: RemoteShutter_CameraPreviewModeEnum, _ fbb: inout FlatBufferBuilder) { fbb.add(element: previewMode.rawValue, def: 0, at: VTOFFSET.previewMode.p) }
   public static func add(exposure: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: exposure, at: VTOFFSET.exposure.p) }
   public static func add(cameraMode: RemoteShutter_RecordingModeEnum, _ fbb: inout FlatBufferBuilder) { fbb.add(element: cameraMode.rawValue, def: 0, at: VTOFFSET.cameraMode.p) }
+  public static func add(supportsPortrait: Bool, _ fbb: inout FlatBufferBuilder) { fbb.add(element: supportsPortrait, def: false,
+   at: VTOFFSET.supportsPortrait.p) }
   public static func endCameraState(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end }
   public static func createCameraState(
     _ fbb: inout FlatBufferBuilder,
@@ -1210,7 +1215,8 @@ public struct RemoteShutter_CameraState: FlatBufferObject, Verifiable {
     activeDeviceIdOffset activeDeviceId: Offset = Offset(),
     previewMode: RemoteShutter_CameraPreviewModeEnum = .unknown,
     exposureOffset exposure: Offset = Offset(),
-    cameraMode: RemoteShutter_RecordingModeEnum = .unknown
+    cameraMode: RemoteShutter_RecordingModeEnum = .unknown,
+    supportsPortrait: Bool = false
   ) -> Offset {
     let __start = RemoteShutter_CameraState.startCameraState(&fbb)
     RemoteShutter_CameraState.add(currentCamera: currentCamera, &fbb)
@@ -1227,6 +1233,7 @@ public struct RemoteShutter_CameraState: FlatBufferObject, Verifiable {
     RemoteShutter_CameraState.add(previewMode: previewMode, &fbb)
     RemoteShutter_CameraState.add(exposure: exposure, &fbb)
     RemoteShutter_CameraState.add(cameraMode: cameraMode, &fbb)
+    RemoteShutter_CameraState.add(supportsPortrait: supportsPortrait, &fbb)
     return RemoteShutter_CameraState.endCameraState(&fbb, start: __start)
   }
 
@@ -1246,6 +1253,7 @@ public struct RemoteShutter_CameraState: FlatBufferObject, Verifiable {
     try _v.visit(field: VTOFFSET.previewMode.p, fieldName: "previewMode", required: false, type: RemoteShutter_CameraPreviewModeEnum.self)
     try _v.visit(field: VTOFFSET.exposure.p, fieldName: "exposure", required: false, type: ForwardOffset<RemoteShutter_ExposureState>.self)
     try _v.visit(field: VTOFFSET.cameraMode.p, fieldName: "cameraMode", required: false, type: RemoteShutter_RecordingModeEnum.self)
+    try _v.visit(field: VTOFFSET.supportsPortrait.p, fieldName: "supportsPortrait", required: false, type: Bool.self)
     _v.finish()
   }
 }

RemoteCam/MultiCamChrome.swift

@@ -80,7 +80,7 @@ enum RigTray {
         switch mode {
         case .video:
             items.append(.resolution)
-        case .photo:
+        case .photo, .portrait:
             items.append(contentsOf: [.format, .hdr])
         }