diff --git a/src/lib/Accessory.spec.ts b/src/lib/Accessory.spec.ts index 296feb96c..ddf9cf724 100644 --- a/src/lib/Accessory.spec.ts +++ b/src/lib/Accessory.spec.ts @@ -34,7 +34,7 @@ import { Units, } from './Characteristic.js' import { createCameraControllerOptions, MOCK_IMAGE } from './controller/CameraController.spec.js' -import { CameraController, Controller, ControllerIdentifier, ControllerServiceMap } from './controller/index.js' +import { CameraController, Controller, ControllerIdentifier, ControllerServiceMap, ResourceRequestReason, SnapshotController } from './controller/index.js' import { HAPHTTPCode, HAPServer, HAPStatus, IdentifyCallback, TLVErrorCode } from './HAPServer.js' import { AccessoryInfo, PairingInformation, PermissionTypes } from './model/AccessoryInfo.js' import { IdentifierCache } from './model/IdentifierCache.js' @@ -2384,6 +2384,23 @@ describe('accessory', () => { expect(callback).toHaveBeenCalledWith(undefined, MOCK_IMAGE) }) + it('retrieve image resource from any snapshot controller', async () => { + accessory.removeController(cameraController) + const snapshotController: SnapshotController = { + controllerId: () => 'snapshot', + constructServices: () => ({}), + initWithServices: () => undefined, + configureServices: () => undefined, + handleControllerRemoved: () => undefined, + handleSnapshotRequest: vi.fn().mockResolvedValue(MOCK_IMAGE), + } + accessory.configureController(snapshotController) + + await testRequestResponse({ reason: ResourceRequestReason.PERIODIC }) + expect(callback).toHaveBeenCalledWith(undefined, MOCK_IMAGE) + expect(snapshotController.handleSnapshotRequest).toHaveBeenCalledWith(200, 200, accessory.displayName, ResourceRequestReason.PERIODIC) + }) + it('retrieve image resource on bridge', async () => { accessory.addBridgedAccessory(new Accessory('Bridged accessory', uuid.generate('bridged'))) accessory._assignIDs(accessory._identifierCache!) diff --git a/src/lib/Accessory.ts b/src/lib/Accessory.ts index ed3f346f4..a459afaa7 100644 --- a/src/lib/Accessory.ts +++ b/src/lib/Accessory.ts @@ -38,12 +38,13 @@ import { Perms, } from './Characteristic.js' import { - CameraController, Controller, ControllerConstructor, ControllerIdentifier, ControllerServiceMap, isSerializableController, + isSnapshotController, + SnapshotController, } from './controller/index.js' import { AccessoriesCallback, @@ -432,7 +433,7 @@ export class Accessory extends EventEmitter { private controllers: Record = {} private serializedControllers?: Record // store uninitialized controller data after a Accessory.deserialize call - private activeCameraController?: CameraController + private activeCameraController?: SnapshotController /** * @private @@ -847,7 +848,7 @@ export class Accessory extends EventEmitter { this.controllers[id] = context - if (controller instanceof CameraController) { // save CameraController for Snapshot handling + if (isSnapshotController(controller)) { // save the controller for snapshot handling this.activeCameraController = controller } } @@ -1911,7 +1912,7 @@ export class Accessory extends EventEmitter { const aid = data.aid // aid is optionally supplied by HomeKit (for example when camera is bridged, multiple cams, etc) let accessory: Accessory | undefined - let controller: CameraController | undefined + let controller: SnapshotController | undefined if (aid) { accessory = this.getAccessoryByAID(aid) if (accessory && accessory.activeCameraController) { diff --git a/src/lib/Characteristic.ts b/src/lib/Characteristic.ts index 38c55a910..14d628f5b 100644 --- a/src/lib/Characteristic.ts +++ b/src/lib/Characteristic.ts @@ -17,8 +17,20 @@ import type { AudioFeedback, BatteryLevel, Brightness, + BufferActivityCommand, + BufferEventCommand, + BufferEventSequenceNumber, + BufferUploadCommand, ButtonEvent, + CameraCapabilitiesConfiguration, + CameraClientCertificate, + CameraClientCertificateStatus, + CameraClientCSR, + CameraKey, + CameraKeyID, CameraOperatingModeIndicator, + CameraRecordingPublishingPoint, + CameraZones, CarbonDioxideDetected, CarbonDioxideLevel, CarbonDioxidePeakLevel, @@ -36,6 +48,7 @@ import type { ConfiguredName, ConnectionHealthMonitor, ContactSensorState, + ContributingSensors, CoolingThresholdTemperature, CryptoHash, CurrentAirPurifierState, @@ -105,6 +118,7 @@ import type { MetricsBufferFullState, Model, MotionDetected, + MotionEnabled, MultifunctionButton, Mute, Name, @@ -153,6 +167,7 @@ import type { RotationDirection, RotationSpeed, RouterStatus, + RTPStreamingControl, Saturation, SecuritySystemAlarmType, SecuritySystemCurrentState, @@ -163,6 +178,7 @@ import type { SelectedRTPStreamConfiguration, SelectedSessionKeepaliveConfigurationList, SelectedSleepConfiguration, + SensorUUID, SerialNumber, ServiceLabelIndex, ServiceLabelNamespace, @@ -189,11 +205,13 @@ import type { StatusJammed, StatusLowBattery, StatusTampered, + StreamingEnabled, StreamingStatus, SulphurDioxideDensity, SupportedAssetTypes, SupportedAudioRecordingConfiguration, SupportedAudioStreamConfiguration, + SupportedAudioStreamTiers, SupportedCameraRecordingConfiguration, SupportedCharacteristicValueTransitionConfiguration, SupportedDataStreamTransportConfiguration, @@ -208,6 +226,7 @@ import type { SupportedTransferTransportConfiguration, SupportedVideoRecordingConfiguration, SupportedVideoStreamConfiguration, + SupportedVideoStreamTiers, SwingMode, TapType, TargetAirPurifierState, @@ -249,6 +268,14 @@ import type { WANConfigurationList, WANStatusList, WaterLevel, + WebRTCNumberOfActiveSessions, + WebRTCProvideAnswer, + WebRTCReoffer, + WebRTCSolicitOffer, + WebRTCStreamingControl, + WebRTCSupportedAudioStreamTiers, + WebRTCSupportedVideoStreamTiers, + WebRTCUpdateSession, WiFiCapabilities, WiFiConfigurationControl, WiFiSatelliteStatus, @@ -815,14 +842,62 @@ export class Characteristic extends EventEmitter { * @group Characteristic Definitions */ public static Brightness: typeof Brightness + /** + * @group Characteristic Definitions + */ + public static BufferActivityCommand: typeof BufferActivityCommand + /** + * @group Characteristic Definitions + */ + public static BufferEventCommand: typeof BufferEventCommand + /** + * @group Characteristic Definitions + */ + public static BufferEventSequenceNumber: typeof BufferEventSequenceNumber + /** + * @group Characteristic Definitions + */ + public static BufferUploadCommand: typeof BufferUploadCommand /** * @group Characteristic Definitions */ public static ButtonEvent: typeof ButtonEvent + /** + * @group Characteristic Definitions + */ + public static CameraCapabilitiesConfiguration: typeof CameraCapabilitiesConfiguration + /** + * @group Characteristic Definitions + */ + public static CameraClientCertificate: typeof CameraClientCertificate + /** + * @group Characteristic Definitions + */ + public static CameraClientCertificateStatus: typeof CameraClientCertificateStatus + /** + * @group Characteristic Definitions + */ + public static CameraClientCSR: typeof CameraClientCSR + /** + * @group Characteristic Definitions + */ + public static CameraKey: typeof CameraKey + /** + * @group Characteristic Definitions + */ + public static CameraKeyID: typeof CameraKeyID /** * @group Characteristic Definitions */ public static CameraOperatingModeIndicator: typeof CameraOperatingModeIndicator + /** + * @group Characteristic Definitions + */ + public static CameraRecordingPublishingPoint: typeof CameraRecordingPublishingPoint + /** + * @group Characteristic Definitions + */ + public static CameraZones: typeof CameraZones /** * @group Characteristic Definitions */ @@ -891,6 +966,10 @@ export class Characteristic extends EventEmitter { * @group Characteristic Definitions */ public static ContactSensorState: typeof ContactSensorState + /** + * @group Characteristic Definitions + */ + public static ContributingSensors: typeof ContributingSensors /** * @group Characteristic Definitions */ @@ -1167,6 +1246,10 @@ export class Characteristic extends EventEmitter { * @group Characteristic Definitions */ public static MotionDetected: typeof MotionDetected + /** + * @group Characteristic Definitions + */ + public static MotionEnabled: typeof MotionEnabled /** * @group Characteristic Definitions */ @@ -1362,6 +1445,10 @@ export class Characteristic extends EventEmitter { * @group Characteristic Definitions */ public static RouterStatus: typeof RouterStatus + /** + * @group Characteristic Definitions + */ + public static RTPStreamingControl: typeof RTPStreamingControl /** * @group Characteristic Definitions */ @@ -1402,6 +1489,10 @@ export class Characteristic extends EventEmitter { * @group Characteristic Definitions */ public static SelectedSleepConfiguration: typeof SelectedSleepConfiguration + /** + * @group Characteristic Definitions + */ + public static SensorUUID: typeof SensorUUID /** * @group Characteristic Definitions */ @@ -1506,6 +1597,10 @@ export class Characteristic extends EventEmitter { * @group Characteristic Definitions */ public static StatusTampered: typeof StatusTampered + /** + * @group Characteristic Definitions + */ + public static StreamingEnabled: typeof StreamingEnabled /** * @group Characteristic Definitions */ @@ -1526,6 +1621,10 @@ export class Characteristic extends EventEmitter { * @group Characteristic Definitions */ public static SupportedAudioStreamConfiguration: typeof SupportedAudioStreamConfiguration + /** + * @group Characteristic Definitions + */ + public static SupportedAudioStreamTiers: typeof SupportedAudioStreamTiers /** * @group Characteristic Definitions */ @@ -1582,6 +1681,10 @@ export class Characteristic extends EventEmitter { * @group Characteristic Definitions */ public static SupportedVideoStreamConfiguration: typeof SupportedVideoStreamConfiguration + /** + * @group Characteristic Definitions + */ + public static SupportedVideoStreamTiers: typeof SupportedVideoStreamTiers /** * @group Characteristic Definitions */ @@ -1752,6 +1855,38 @@ export class Characteristic extends EventEmitter { * @group Characteristic Definitions */ public static WaterLevel: typeof WaterLevel + /** + * @group Characteristic Definitions + */ + public static WebRTCNumberOfActiveSessions: typeof WebRTCNumberOfActiveSessions + /** + * @group Characteristic Definitions + */ + public static WebRTCProvideAnswer: typeof WebRTCProvideAnswer + /** + * @group Characteristic Definitions + */ + public static WebRTCReoffer: typeof WebRTCReoffer + /** + * @group Characteristic Definitions + */ + public static WebRTCSolicitOffer: typeof WebRTCSolicitOffer + /** + * @group Characteristic Definitions + */ + public static WebRTCStreamingControl: typeof WebRTCStreamingControl + /** + * @group Characteristic Definitions + */ + public static WebRTCSupportedAudioStreamTiers: typeof WebRTCSupportedAudioStreamTiers + /** + * @group Characteristic Definitions + */ + public static WebRTCSupportedVideoStreamTiers: typeof WebRTCSupportedVideoStreamTiers + /** + * @group Characteristic Definitions + */ + public static WebRTCUpdateSession: typeof WebRTCUpdateSession /** * @group Characteristic Definitions */ diff --git a/src/lib/Service.ts b/src/lib/Service.ts index bf829c2ac..a244343d6 100644 --- a/src/lib/Service.ts +++ b/src/lib/Service.ts @@ -11,9 +11,17 @@ import type { Assistant, AudioStreamManagement, Battery, + CameraBufferManagement, + CameraCapabilities, + CameraClientCertificateManagement, + CameraGlobalOperatingMode, + CameraKeyManagement, + CameraMotionZones, + CameraMultiTierRTPStreamManagement, CameraOperatingMode, CameraRecordingManagement, CameraRTPStreamManagement, + CameraWebRTCStreamManagement, CarbonDioxideSensor, CarbonMonoxideSensor, CloudRelay, @@ -218,6 +226,34 @@ export class Service extends EventEmitter { * @group Service Definitions */ public static Battery: typeof Battery + /** + * @group Service Definitions + */ + public static CameraBufferManagement: typeof CameraBufferManagement + /** + * @group Service Definitions + */ + public static CameraCapabilities: typeof CameraCapabilities + /** + * @group Service Definitions + */ + public static CameraClientCertificateManagement: typeof CameraClientCertificateManagement + /** + * @group Service Definitions + */ + public static CameraGlobalOperatingMode: typeof CameraGlobalOperatingMode + /** + * @group Service Definitions + */ + public static CameraKeyManagement: typeof CameraKeyManagement + /** + * @group Service Definitions + */ + public static CameraMotionZones: typeof CameraMotionZones + /** + * @group Service Definitions + */ + public static CameraMultiTierRTPStreamManagement: typeof CameraMultiTierRTPStreamManagement /** * @group Service Definitions */ @@ -230,6 +266,10 @@ export class Service extends EventEmitter { * @group Service Definitions */ public static CameraRTPStreamManagement: typeof CameraRTPStreamManagement + /** + * @group Service Definitions + */ + public static CameraWebRTCStreamManagement: typeof CameraWebRTCStreamManagement /** * @group Service Definitions */ diff --git a/src/lib/camera/CMAFIngest.spec.ts b/src/lib/camera/CMAFIngest.spec.ts new file mode 100644 index 000000000..b1ad97a96 --- /dev/null +++ b/src/lib/camera/CMAFIngest.spec.ts @@ -0,0 +1,455 @@ +import type { CMAFRecordingDelegate, CMAFSegment, SecureVideoIngestCredentials } from '../controller/SecureVideoController.js' +import type { CMAFIngestContext } from './CMAFIngest.js' +import type { CMAFTrackSegment } from './CMAFTracks.js' + +import { Buffer } from 'node:buffer' +import { EventEmitter } from 'node:events' +import http2 from 'node:http2' + +import { afterEach, describe, expect, it, vi } from 'vitest' + +import { initSegment, mediaSegment } from '../../test-utils/fmp4.js' +import { buildMediaPlaylist, buildMultivariantPlaylist, CMAFIngest, formatDuration, formatUserAgent, mapStatus } from './CMAFIngest.js' +import { splitInitSegment, splitMediaSegment } from './CMAFTracks.js' +import { BufferUploadCommandType, BufferUploadStopAction, CMAFError } from './SecureVideoTypes.js' + +const VIDEO = { id: 1, handler: 'vide' as const, timescale: 90000 } +const AUDIO = { + id: 2, + handler: 'soun' as const, + timescale: 32000, + defaultSampleDuration: 1024, +} +const BASE = '/v1/token/' + +interface Request { + path: string + contentType: string + userAgent?: string + extinf?: string + body: string +} + +function fragment(seconds: number, nonSyncStart = false): Buffer { + return mediaSegment(1, [ + { + trackId: 1, + durations: [seconds * 45000, seconds * 45000], + samples: [ + Buffer.alloc(100, 1), + Buffer.alloc(100, 1), + ], + nonSyncStart, + }, + { + trackId: 2, + durations: [1024], + samples: [ + Buffer.alloc(20, 2), + ], + }, + ]) +} + +function fakeClient(requests: Request[], status: (path: string, attempt: number) => number, attempts: Map) { + return { + on: vi.fn(), + closed: false, + destroyed: false, + close: vi.fn(), + request: (headers: http2.OutgoingHttpHeaders) => { + const stream = new EventEmitter() as EventEmitter & { end(body: Buffer): void } + stream.end = (body) => { + const path = String(headers[':path']).slice(BASE.length) + const attempt = attempts.get(path) ?? 0 + attempts.set(path, attempt + 1) + requests.push({ + path, + contentType: String(headers['content-type']), + userAgent: headers['user-agent'] as string | undefined, + extinf: headers['x-extinf-duration'] as string | undefined, + body: body.toString('latin1'), + }) + setImmediate(() => { + stream.emit('response', { + ':status': status(path, attempt), + 'x-apple-request-uuid': '4f1c', + }) + stream.emit('end') + }) + } + return stream + }, + } as unknown as http2.ClientHttp2Session +} + +function setup(segments: CMAFSegment[], status: (path: string, attempt: number) => number = () => 201) { + const requests: Request[] = [] + const events: string[] = [] + const errors: string[] = [] + const attempts = new Map() + let sessionNumber = 0n + const connect = vi.spyOn(http2, 'connect').mockImplementation(() => fakeClient(requests, status, attempts)) + + let resolveDone!: () => void + const done = new Promise((resolve) => { + resolveDone = resolve + }) + const delegate: CMAFRecordingDelegate = { + async* streamClip() { + try { + yield* segments + } finally { + // the generator is closed in the upload's cleanup, after the last request; give the event loop a few turns + setTimeout(resolveDone, 20) + } + }, + } + const context: CMAFIngestContext = { + userAgent: () => 'test-agent/1.0', + credentials: () => ({ + publishingPoint: { url: `https://ingest.example.com${BASE}`, serverCACertificates: [] }, + privateKey: 'key', + clientCertificate: Buffer.from('cert'), + keys: [], + }) as unknown as SecureVideoIngestCredentials, + nextSessionNumber: () => ++sessionNumber, + reportSessionStart: number => events.push(`start ${number}`), + reportSessionStop: number => events.push(`stop ${number}`), + reportError: (number, error, detail) => { + events.push(`error ${number} ${error}`) + errors.push(detail ?? '') + }, + } + + const ingest = new CMAFIngest(delegate, context) + const clipId = ingest.handleUploadCommand({ sessionId: 99n, command: BufferUploadCommandType.START_AND_STOP }) + return { + requests, + events, + errors, + clipId, + done, + connects: () => connect.mock.calls.length, + } +} + +function videoSegments(requests: Request[], session: number) { + return requests.filter(request => request.path.startsWith(`session_${session}/high/segment_`)).length +} + +afterEach(() => { + vi.restoreAllMocks() +}) + +describe('cMAFIngest', () => { + it('builds the multivariant playlist in the form of the iCloud ingest sample', () => { + const tracks = splitInitSegment(initSegment([VIDEO, AUDIO])) + const segments = [ + splitMediaSegment(fragment(4), tracks), + splitMediaSegment(fragment(2), tracks), + ] + const rate = (part: CMAFTrackSegment) => part.data.length * 8 / part.duration + const average = (parts: CMAFTrackSegment[]) => + parts.reduce((sum, part) => sum + part.data.length, 0) * 8 / parts.reduce((sum, part) => sum + part.duration, 0) + const [video, audio] = [0, 1].map(index => segments.map(parts => parts[index])) + const peak = Math.ceil(Math.max(...video.map(rate)) + Math.max(...audio.map(rate))) + const mean = Math.ceil(average(video) + average(audio)) + + expect(buildMultivariantPlaylist(tracks, segments).split('\n')).toEqual([ + '#EXTM3U', + '#EXT-X-VERSION:11', + '#EXT-X-INDEPENDENT-SEGMENTS', + '', + '#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="aud",NAME="Camera Audio",DEFAULT=YES,CHANNELS="1",URI="audio1/index.m3u8"', + '', + `#EXT-X-STREAM-INF:BANDWIDTH=${peak},AVERAGE-BANDWIDTH=${mean},RESOLUTION=1920x1080,CODECS="hvc1.1.6.L93.B0,mp4a.40.2",AUDIO="aud"`, + 'high/index.m3u8', + '', + ]) + }) + + it('lists one variant per video track, highest first', () => { + const tracks = splitInitSegment(initSegment([ + { + ...VIDEO, + id: 3, + width: 640, + height: 360, + }, + VIDEO, + ])) + const lines = buildMultivariantPlaylist(tracks, []).split('\n') + + expect(lines.filter(line => !line.startsWith('#') && line)).toEqual(['high/index.m3u8', 'mid/index.m3u8']) + expect(lines.some(line => line.startsWith('#EXT-X-MEDIA') || line.includes('AUDIO='))).toBe(false) + }) + + it('formats the user agent of the iCloud ingest', () => { + expect(formatUserAgent({ + software: 'camera.ui-homekit', + vendor: 'Acme', + model: 'Cam (2); 4K', + firmwareVersion: 'v1:2', + })) + .toBe('HKSV/3.0 camera.ui-homekit (Acme; Cam 2 4K; v1 2)') + expect(formatUserAgent({ + software: 'Home Helper 1.0', + vendor: '', + model: 'G234', + firmwareVersion: '4.56.21', + })) + .toBe('HKSV/3.0 Home-Helper-1.0 (unknown; G234; 4.56.21)') + }) + + it('builds the initial and the final media playlist', () => { + const [video] = splitInitSegment(initSegment([VIDEO])) + + expect(buildMediaPlaylist(video, 5).split('\n')).toEqual([ + '#EXTM3U', + '#EXT-X-VERSION:11', + '#EXT-X-TARGETDURATION:5', + '#EXT-X-PLAYLIST-TYPE:EVENT', + '#EXT-X-INDEPENDENT-SEGMENTS', + '#EXT-X-MAP:URI="high.init"', + '', + ]) + + const final = buildMediaPlaylist(video, 5, [4, 6.4]).split('\n') + expect(final).toContain('#EXT-X-TARGETDURATION:6') + expect(final.slice(final.indexOf('#EXTINF:4.00000,'))).toEqual([ + '#EXTINF:4.00000,', + 'segment_1001.m4s', + '#EXTINF:6.40000,', + 'segment_1002.m4s', + '#EXT-X-ENDLIST', + '', + ]) + }) + + it('formats durations with five decimals', () => { + expect(formatDuration(4)).toBe('4.00000') + expect(formatDuration(4.123456)).toBe('4.12346') + }) + + it('uploads a clip in the order of the Matter HLS ingest', async () => { + const { requests, events, clipId, done } = setup([ + { + type: 'init', + data: initSegment([VIDEO, AUDIO]), + }, + { + type: 'media', + data: fragment(4), + }, + { + type: 'media', + data: fragment(4), + last: true, + }, + ]) + await done + + expect(requests.map(({ path, contentType, extinf }) => `${path} ${contentType}${extinf + ? ` ${extinf}` + : ''}`)).toEqual([ + 'session_1/index.m3u8 application/vnd.apple.mpegurl', + 'session_1/high/index.m3u8 application/vnd.apple.mpegurl', + 'session_1/audio1/index.m3u8 application/vnd.apple.mpegurl', + 'session_1/high/high.init video/mp4', + 'session_1/audio1/audio1.init video/mp4', + 'session_1/high/segment_1001.m4s video/iso.segment 4.00000', + 'session_1/audio1/segment_1001.m4s video/iso.segment 0.03200', + 'session_1/high/segment_1002.m4s video/iso.segment 4.00000', + 'session_1/audio1/segment_1002.m4s video/iso.segment 0.03200', + 'session_1/high/index.m3u8 application/vnd.apple.mpegurl', + 'session_1/audio1/index.m3u8 application/vnd.apple.mpegurl', + ]) + expect(requests.every(request => request.userAgent === 'test-agent/1.0')).toBe(true) + expect(requests[1].body).not.toContain('#EXT-X-ENDLIST') + expect(requests[9].body).toContain('segment_1002.m4s\n#EXT-X-ENDLIST') + expect(events).toEqual(['start 1', 'stop 1']) + // the hub links the upload to its clip through the first CMAF session number + expect(clipId).toBe(1n) + }) + + it('continues a recording longer than five minutes in the next session', async () => { + const { requests, events, done } = setup([ + { + type: 'init', + data: initSegment([VIDEO, AUDIO]), + }, + { + type: 'media', + data: fragment(100), + }, + { + type: 'media', + data: fragment(100), + }, + { + type: 'media', + data: fragment(90), + }, + { + type: 'media', + data: fragment(100), + }, + { + type: 'media', + data: fragment(100), + last: true, + }, + ]) + await done + + expect(videoSegments(requests, 1)).toBe(3) + expect(videoSegments(requests, 2)).toBe(2) + const paths = requests.map(request => request.path) + expect(paths.indexOf('session_2/index.m3u8')).toBeGreaterThan(paths.lastIndexOf('session_1/audio1/index.m3u8')) + expect(events).toEqual([ + 'start 1', + 'stop 1', + 'start 2', + 'stop 2', + ]) + }) + + it('never leaves a session with a single segment', async () => { + const { requests, done } = setup([ + { + type: 'init', + data: initSegment([VIDEO, AUDIO]), + }, + { + type: 'media', + data: fragment(100), + }, + { + type: 'media', + data: fragment(100), + }, + { + type: 'media', + data: fragment(50), + }, + { + type: 'media', + data: fragment(100), + last: true, + }, + ]) + await done + + // 100 + 100 + 50 fit, but the last 100 would then form a session of its own + expect(videoSegments(requests, 1)).toBe(2) + expect(videoSegments(requests, 2)).toBe(2) + }) + + it('lets a fragment without a sync sample continue the segment before it', async () => { + const { requests, done } = setup([ + { + type: 'init', + data: initSegment([VIDEO, AUDIO]), + }, + { + type: 'media', + data: fragment(2), + }, + { + type: 'media', + data: fragment(0.5, true), + }, + { + type: 'media', + data: fragment(2), + last: true, + }, + ]) + await done + + const video = requests.filter(request => request.path.startsWith('session_1/high/segment_')) + expect(video.map(request => `${request.path.split('/').pop()} ${request.extinf}`)).toEqual(['segment_1001.m4s 2.50000', 'segment_1002.m4s 2.00000']) + expect(video[0].body.split('moof').length - 1).toBe(2) + }) + + it('sends a request again on a new connection after a server error', async () => { + const { requests, events, connects, done } = setup([ + { + type: 'init', + data: initSegment([VIDEO, AUDIO]), + }, + { + type: 'media', + data: fragment(4), + last: true, + }, + ], (path, attempt) => (path === 'session_1/index.m3u8' && attempt === 0 + ? 503 + : 201)) + await done + + expect(requests.filter(request => request.path === 'session_1/index.m3u8')).toHaveLength(2) + expect(connects()).toBe(2) + expect(events).toEqual(['start 1', 'stop 1']) + }) + + it('reports the first refusal once and closes the opened tracks', async () => { + const { requests, events, errors, done } = setup([ + { + type: 'init', + data: initSegment([VIDEO, AUDIO]), + }, + { + type: 'media', + data: fragment(4), + }, + { + type: 'media', + data: fragment(4), + last: true, + }, + ], path => (path.endsWith('segment_1001.m4s') + ? 400 + : 201)) + await done + + expect(events).toEqual(['start 1', `error 1 ${CMAFError.HTTP_BAD_REQUEST}`]) + expect(errors[0]).toMatch(/^PUT session_1\/(high|audio1)\/segment_1001\.m4s \(\d+ bytes\) -> 400 x-apple-request-uuid=4f1c$/) + // a 4xx is not sent again + expect(requests.filter(request => request.path === 'session_1/high/segment_1001.m4s')).toHaveLength(1) + // the server saw the session begin, so its tracks are closed even though the upload failed + expect(requests.slice(-2).map(request => request.path)).toEqual(['session_1/high/index.m3u8', 'session_1/audio1/index.m3u8']) + expect(requests.at(-1)!.body).toContain('#EXT-X-ENDLIST') + }) + + it('passes the stop command to the delegate', () => { + const handleUploadStop = vi.fn() + const delegate = { + streamClip: vi.fn(), + handleUploadStop, + } as unknown as CMAFRecordingDelegate + const ingest = new CMAFIngest(delegate, {} as CMAFIngestContext) + const stop = { + sessionId: 7n, + command: BufferUploadCommandType.STOP, + stop: 123n, + stopAction: BufferUploadStopAction.FINALIZE, + } + + expect(ingest.handleUploadCommand(stop)).toBe(0n) + expect(handleUploadStop).toHaveBeenCalledWith(stop) + }) + + it('maps http statuses to cmaf errors', () => { + expect(mapStatus(400)).toBe(CMAFError.HTTP_BAD_REQUEST) + expect(mapStatus(401)).toBe(CMAFError.HTTP_INVALID_TOKEN) + expect(mapStatus(403)).toBe(CMAFError.HTTP_INVALID_TOKEN) + expect(mapStatus(404)).toBe(CMAFError.HTTP_NOT_FOUND) + expect(mapStatus(412)).toBe(CMAFError.HTTP_INIT_MISSING) + expect(mapStatus(415)).toBe(CMAFError.HTTP_UNSUPPORTED_MEDIA_TYPE) + expect(mapStatus(503)).toBe(CMAFError.HTTP_SERVICE_UNAVAILABLE) + expect(mapStatus(500)).toBe(CMAFError.HTTP_INTERNAL_SERVER_ERROR) + expect(mapStatus(418)).toBe(CMAFError.UNKNOWN) + }) +}) diff --git a/src/lib/camera/CMAFIngest.ts b/src/lib/camera/CMAFIngest.ts new file mode 100644 index 000000000..65bcd6107 --- /dev/null +++ b/src/lib/camera/CMAFIngest.ts @@ -0,0 +1,658 @@ +import type { CMAFRecordingDelegate, CMAFSegment, SecureVideoIngestCredentials, SecureVideoUserAgent } from '../controller/SecureVideoController.js' +import type { CMAFTrack, CMAFTrackSegment } from './CMAFTracks.js' +import type { BufferUploadCommandRequest } from './SecureVideoTypes.js' + +import { Buffer } from 'node:buffer' +import http2 from 'node:http2' + +import createDebug from 'debug' + +import { splitInitSegment, splitMediaSegment } from './CMAFTracks.js' +import { BufferUploadCommandType, CMAFError } from './SecureVideoTypes.js' + +const debug = createDebug('HAP-NodeJS:SecureVideo:Ingest') + +const FIRST_SEGMENT_NUMBER = 1001 +const DEFAULT_SEGMENT_DURATION_SECONDS = 2 +// Matter 1.6.1 Push AV Stream Transport 11.7.1.6: a session lasts at most five minutes and at least two segments +const MAX_SESSION_SECONDS = 300 +// DASH-IF ingest 5.3: a failed transfer is sent again on a new connection +const MAX_RETRIES = 2 +const RETRY_DELAY_MS = 500 +const PLAYLIST_CONTENT_TYPE = 'application/vnd.apple.mpegurl' +const INIT_CONTENT_TYPE = 'video/mp4' +const SEGMENT_CONTENT_TYPE = 'video/iso.segment' +// the playlist form of the iCloud ingest sample +const HLS_VERSION = 11 +const AUDIO_GROUP = 'aud' +const AUDIO_NAME = 'Camera Audio' +const USER_AGENT_PRODUCT = 'HKSV/3.0' + +/** + * Everything the ingest needs from the controller: the negotiated credentials, the persisted session number and the + * event reporters. Keeps the credential and event handling internal to the controller. + * + * @group Camera Secure Video + */ +export interface CMAFIngestContext { + /** the User-Agent of the next upload, see {@link formatUserAgent} */ + userAgent(): string + credentials(): SecureVideoIngestCredentials + /** the number of the next CMAF session: starts at 1 for a new publishing point, persisted, never reused */ + nextSessionNumber(): bigint + reportSessionStart(cmafSessionId: bigint): void + reportSessionStop(cmafSessionId: bigint): void + /** `detail` names the failed request or the transport error, for logging only */ + reportError(cmafSessionId: bigint, error: CMAFError, detail?: string): void +} + +interface UploadSession { + /** number of the first CMAF session of the clip, the hub learns it from the upload command response */ + clipId: bigint + abort: AbortController + /** no further hub command is expected, the session is dropped once the upload ends */ + finalize: boolean + /** the hub sent stop, the upload ends with the fragments already received */ + stopRequested: boolean + finished: boolean +} + +interface CMAFSession { + number: bigint + /** publishing point path plus `session_/` */ + prefix: string + startedAt: number + seconds: number + /** uploaded segment durations per track id */ + durations: Map + /** tracks whose initial media playlist was accepted; only those get a final playlist */ + opened: Set + failed: boolean + ended: boolean +} + +interface PreparedSegment { + parts: CMAFTrackSegment[] + /** seconds, the longest track */ + duration: number +} + +interface PutResult { + status: number + /** first bytes of the response body, the ingest edge explains rejections there */ + body: string + /** response headers as name=value pairs, they carry the request id Apple asks for */ + headers: string +} + +interface Bitrate { + peak: number + average: number +} + +class IngestConnection { + private client?: http2.ClientHttp2Session + + constructor(private readonly origin: string, private readonly options: http2.SecureClientSessionOptions, readonly userAgent: string) {} + + session(): http2.ClientHttp2Session { + if (!this.client || this.client.closed || this.client.destroyed) { + this.client = http2.connect(this.origin, this.options) + this.client.on('error', error => debug('http2 session error: %s', error.message)) + } + return this.client + } + + /** closes the current connection once its streams are done, the next request opens a new one */ + renew(): void { + this.client?.close() + this.client = undefined + } +} + +function put(client: http2.ClientHttp2Session, path: string, body: Buffer, contentType: string, headers: http2.OutgoingHttpHeaders): Promise { + return new Promise((resolve, reject) => { + const request = client.request({ + ':method': 'PUT', + ':path': path, + 'content-type': contentType, + 'content-length': body.length, + ...headers, + }) + + let status = 0 + let summary = '' + const chunks: Buffer[] = [] + request.on('response', (responseHeaders) => { + status = Number(responseHeaders[':status']) + summary = Object.entries(responseHeaders) + .filter(([name]) => !name.startsWith(':')) + .map(([name, value]) => `${name}=${Array.isArray(value) + ? value.join(',') + : value}`) + .join(' ') + }) + request.on('data', (chunk: Buffer) => { + chunks.push(chunk) + }) + request.on('end', () => { + const response = Buffer.concat(chunks).toString('utf8').slice(0, 512) + debug('PUT %s (%d bytes) -> %d %s %s', path.slice(path.indexOf('session_')), body.length, status, response, summary) + resolve({ status, body: response, headers: summary }) + }) + request.on('error', reject) + request.end(body) + }) +} + +// appends the parts of a segment to the one before it, a CMAF segment may hold several fragments +function appendSegment(target: PreparedSegment, source: PreparedSegment): void { + for (const part of source.parts) { + const index = target.parts.findIndex(existing => existing.track === part.track) + if (index < 0) { + target.parts.push(part) + continue + } + const existing = target.parts[index] + target.parts[index] = { + ...existing, + data: Buffer.concat([existing.data, part.data]), + duration: existing.duration + part.duration, + samples: existing.samples + part.samples, + } + } + target.duration = target.parts.reduce((longest, part) => Math.max(longest, part.duration), 0) +} + +function sleep(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)) +} + +/** + * Formats a segment duration for the `#EXTINF` tag and the `X-EXTINF-duration` header. + * + * @group Camera Secure Video + */ +export function formatDuration(seconds: number): string { + return seconds.toFixed(5) +} + +function quote(value: string): string { + return `"${value}"` +} + +// RFC 8216bis 4.4.6.2: BANDWIDTH is the peak segment bit rate, AVERAGE-BANDWIDTH the average one +function bitrate(track: CMAFTrack, segments: CMAFTrackSegment[][]): Bitrate { + let peak = 0 + let bytes = 0 + let seconds = 0 + for (const part of segments.flat()) { + if (part.track !== track || part.duration <= 0) { + continue + } + peak = Math.max(peak, part.data.length * 8 / part.duration) + bytes += part.data.length + seconds += part.duration + } + return { + peak, + average: seconds > 0 + ? bytes * 8 / seconds + : 0, + } +} + +// the vendor sub-fields may hold spaces, dashes and underscores, but no colons, semicolons or parentheses +function userAgentField(value: string): string { + return value.replace(/[^\x20-\x7E]|[:;()]/g, ' ').replace(/\s+/g, ' ').trim() || 'unknown' +} + +/** + * Formats the User-Agent the iCloud ingest requires on every request: + * `HKSV/3.0 (; ; )`. + * + * @group Camera Secure Video + */ +export function formatUserAgent(agent: SecureVideoUserAgent): string { + const software = agent.software.replace(/[^\w.-]+/g, '-') || 'unknown' + return `${USER_AGENT_PRODUCT} ${software} (${[agent.vendor, agent.model, agent.firmwareVersion].map(userAgentField).join('; ')})` +} + +/** + * Builds the HLS multivariant playlist of a CMAF session in the form of the iCloud ingest sample: one variant per video + * track, highest first, and the audio tracks as renditions of one group. The bit rates come from the segments at hand + * when the session starts and include the audio of the variant. + * + * @group Camera Secure Video + */ +export function buildMultivariantPlaylist(tracks: CMAFTrack[], segments: CMAFTrackSegment[][]): string { + const video = tracks.filter(track => track.kind === 'video') + if (!video.length) { + throw new Error('a CMAF session needs a video track') + } + const audio = tracks.filter(track => track.kind === 'audio') + const audioBitrates = audio.map(track => bitrate(track, segments)) + const audioPeak = Math.max(0, ...audioBitrates.map(entry => entry.peak)) + const audioAverage = Math.max(0, ...audioBitrates.map(entry => entry.average)) + + const lines = [ + '#EXTM3U', + `#EXT-X-VERSION:${HLS_VERSION}`, + '#EXT-X-INDEPENDENT-SEGMENTS', + '', + ] + if (audio.length) { + audio.forEach((track, index) => { + const attributes = [ + 'TYPE=AUDIO', + `GROUP-ID=${quote(AUDIO_GROUP)}`, + `NAME=${quote(index === 0 + ? AUDIO_NAME + : `${AUDIO_NAME} ${index + 1}`)}`, + `DEFAULT=${index === 0 + ? 'YES' + : 'NO'}`, + ...(track.channels + ? [`CHANNELS=${quote(String(track.channels))}`] + : []), + `URI=${quote(`${track.name}/index.m3u8`)}`, + ] + lines.push(`#EXT-X-MEDIA:${attributes.join(',')}`) + }) + lines.push('') + } + + for (const track of video) { + const { peak, average } = bitrate(track, segments) + const codecs = [ + track.codecs, + ...audio.slice(0, 1).map(entry => entry.codecs), + ] + const attributes = [ + `BANDWIDTH=${Math.ceil(peak + audioPeak)}`, + `AVERAGE-BANDWIDTH=${Math.ceil(average + audioAverage)}`, + ...(track.width && track.height + ? [`RESOLUTION=${track.width}x${track.height}`] + : []), + // CODECS lists every codec of the variant, it is left out when one of them is unknown + ...(codecs.every(Boolean) + ? [`CODECS=${quote(codecs.join(','))}`] + : []), + ...(audio.length + ? [`AUDIO=${quote(AUDIO_GROUP)}`] + : []), + ] + lines.push(`#EXT-X-STREAM-INF:${attributes.join(',')}`, `${track.name}/index.m3u8`, '') + } + return lines.join('\n') +} + +/** + * Builds the HLS media playlist of a track. Without durations it is the initial playlist that opens the track, with + * them the final one that lists every uploaded segment and closes it (Matter 1.6.1 11.7.1.8.2). + * + * @group Camera Secure Video + */ +export function buildMediaPlaylist(track: CMAFTrack, targetDuration: number, durations?: number[]): string { + const target = Math.max(targetDuration, ...(durations ?? []).map(Math.round)) + const lines = [ + '#EXTM3U', + `#EXT-X-VERSION:${HLS_VERSION}`, + `#EXT-X-TARGETDURATION:${target}`, + '#EXT-X-PLAYLIST-TYPE:EVENT', + '#EXT-X-INDEPENDENT-SEGMENTS', + `#EXT-X-MAP:URI=${quote(`${track.name}.init`)}`, + ] + if (durations) { + durations.forEach((duration, index) => { + lines.push(`#EXTINF:${formatDuration(duration)},`, `segment_${FIRST_SEGMENT_NUMBER + index}.m4s`) + }) + lines.push('#EXT-X-ENDLIST') + } + lines.push('') + return lines.join('\n') +} + +function trailingSlash(pathname: string): string { + return pathname.endsWith('/') + ? pathname + : `${pathname}/` +} + +function toPem(der: Buffer): string { + const lines = der.toString('base64').match(/.{1,64}/g) ?? [] + return `-----BEGIN CERTIFICATE-----\n${lines.join('\n')}\n-----END CERTIFICATE-----\n` +} + +/** + * Maps a publishing point HTTP status to the CMAF error reported to the HomeKit HomeHub. + * + * @group Camera Secure Video + */ +export function mapStatus(status: number): CMAFError { + switch (status) { + case 400: + return CMAFError.HTTP_BAD_REQUEST + case 401: + case 403: + return CMAFError.HTTP_INVALID_TOKEN + case 404: + return CMAFError.HTTP_NOT_FOUND + case 412: + return CMAFError.HTTP_INIT_MISSING + case 415: + return CMAFError.HTTP_UNSUPPORTED_MEDIA_TYPE + case 503: + return CMAFError.HTTP_SERVICE_UNAVAILABLE + default: + return status >= 500 + ? CMAFError.HTTP_INTERNAL_SERVER_ERROR + : CMAFError.UNKNOWN + } +} + +/** + * Uploads HomeKit Secure Video clips to the CMAF publishing point following DASH-IF ingest interface 2 for HLS with the + * deviations of the Matter 1.6.1 Push AV Stream Transport cluster. The media is supplied by a + * {@link CMAFRecordingDelegate} as muxed fMP4; this splits it into one CMAF track per media type and PUTs the playlists + * and segments over HTTP/2 with the client certificate the controller manages. Segments are uploaded in the clear. + * + * @group Camera Secure Video + */ +export class CMAFIngest { + private readonly sessions = new Map() + + constructor(private readonly delegate: CMAFRecordingDelegate, private readonly context: CMAFIngestContext) {} + + handleUploadCommand(request: BufferUploadCommandRequest): bigint { + const existing = this.sessions.get(request.sessionId) + + if (request.command === BufferUploadCommandType.STOP) { + this.delegate.handleUploadStop?.(request) + if (existing) { + existing.finalize = true + existing.stopRequested = true + if (existing.finished) { + this.sessions.delete(request.sessionId) + } + return existing.clipId + } + debug('stop for unknown session %s', request.sessionId) + return 0n + } + + if (existing) { + return existing.clipId + } + + const session: UploadSession = { + clipId: this.context.nextSessionNumber(), + abort: new AbortController(), + finalize: request.command === BufferUploadCommandType.START_AND_STOP, + stopRequested: false, + finished: false, + } + this.sessions.set(request.sessionId, session) + + // the session ends with the hub's stop command, which may arrive after the upload itself ended + this.upload(request, session).finally(() => { + session.finished = true + if (session.finalize) { + this.sessions.delete(request.sessionId) + } + }) + return session.clipId + } + + destroy(): void { + for (const session of this.sessions.values()) { + session.abort.abort() + } + this.sessions.clear() + } + + private async upload(request: BufferUploadCommandRequest, session: UploadSession): Promise { + const { publishingPoint, privateKey, clientCertificate, ca } = this.context.credentials() + if (!publishingPoint || !privateKey || !clientCertificate) { + debug('upload ignored, ingest is not provisioned') + this.context.reportError(session.clipId, CMAFError.INVALID_STATE) + return + } + + const url = new URL(publishingPoint.url) + const base = trailingSlash(url.pathname) + const connection = new IngestConnection(url.origin, { + key: privateKey, + cert: [ + toPem(clientCertificate), + ...(ca + ? [ + toPem(ca), + ] + : []), + ].join(''), + ca: publishingPoint.serverCACertificates.map(toPem), + }, this.context.userAgent()) + + const clip = this.delegate.streamClip({ command: request, signal: session.abort.signal }) + let tracks: CMAFTrack[] = [] + let targetDuration = 0 + let current: CMAFSession | undefined + try { + const first = await clip.next() + if (first.done || first.value.type !== 'init') { + throw new Error('delegate did not yield an init segment first') + } + tracks = splitInitSegment(first.value.data) + if (!tracks.some(track => track.kind === 'video')) { + throw new Error('init segment without a video track') + } + + const prepare = (segment: CMAFSegment): PreparedSegment => { + const fallback = segment.duration ?? DEFAULT_SEGMENT_DURATION_SECONDS + const parts = splitMediaSegment(segment.data, tracks).map(part => (part.duration > 0 + ? part + : { ...part, duration: fallback })) + return { + parts, + duration: parts.reduce((longest, part) => Math.max(longest, part.duration), 0), + } + } + + // two segments of lookahead: whether the segment after the current one is the last decides where a session ends + const window: PreparedSegment[] = [] + let ended = false + const fill = async () => { + while (!ended && window.length < 3) { + if (session.stopRequested) { + ended = true + break + } + const next = await clip.next() + if (next.done) { + ended = true + } else if (next.value.type === 'media') { + const prepared = prepare(next.value) + const video = prepared.parts.find(part => part.track.kind === 'video') + const previous = window.at(-1) + // every segment starts with a sync sample (CMAF, EXT-X-INDEPENDENT-SEGMENTS), a fragment that does not + // continues the segment before it + if (previous && video && !video.startsWithSync) { + appendSegment(previous, prepared) + } else { + window.push(prepared) + } + ended = !!next.value.last + } + } + } + + let reserved: bigint | undefined = session.clipId + for (;;) { + await fill() + const segment = window.shift() + if (!segment) { + break + } + + const lastFollower = ended && window.length === 1 + ? window[0] + : undefined + if (current && this.endsBefore(current, segment, lastFollower)) { + await this.endSession(connection, current, tracks, targetDuration) + if (!current.ended) { + return + } + current = undefined + } + if (!current) { + // the target duration must not change once announced, the headroom absorbs keyframe jitter + targetDuration ||= Math.ceil(segment.duration) + 1 + const known = [segment, ...window].map(entry => entry.parts) + current = await this.beginSession(connection, base, tracks, known, targetDuration, reserved ?? this.context.nextSessionNumber()) + reserved = undefined + if (current.failed) { + return + } + } + + const active = current + await Promise.all(segment.parts.map(part => this.putSegment(connection, active, part))) + if (active.failed) { + return + } + active.seconds += segment.duration + } + + if (current) { + await this.endSession(connection, current, tracks, targetDuration) + } + debug('upload of clip %s finished', session.clipId) + } catch (error) { + const message = error instanceof Error + ? error.message + : String(error) + debug('upload of clip %s failed: %s', session.clipId, message) + if (!current?.failed) { + this.context.reportError(current?.number ?? session.clipId, CMAFError.CONNECTION_FAILED, message) + } + if (current) { + current.failed = true + } + } finally { + // a session the server saw begin is closed even after a failure, so it is not left half open on the server side + if (current && !current.ended) { + await this.closeTracks(connection, current, tracks, targetDuration).catch(() => undefined) + } + await clip.return?.(undefined).catch(() => undefined) + connection.renew() + } + } + + // a session ends after five minutes; a last segment that would not fit takes the current one along, so no session + // is left with a single segment + private endsBefore(session: CMAFSession, segment: PreparedSegment, lastFollower?: PreparedSegment): boolean { + if (Date.now() - session.startedAt >= MAX_SESSION_SECONDS * 1000 || session.seconds + segment.duration > MAX_SESSION_SECONDS) { + return true + } + return !!lastFollower && session.seconds + segment.duration + lastFollower.duration > MAX_SESSION_SECONDS + } + + private async beginSession(connection: IngestConnection, base: string, tracks: CMAFTrack[], known: CMAFTrackSegment[][], targetDuration: number, + number: bigint): Promise { + const session: CMAFSession = { + number, + prefix: `${base}session_${number}/`, + startedAt: Date.now(), + seconds: 0, + durations: new Map(tracks.map(track => [track.id, []])), + opened: new Set(), + failed: false, + ended: false, + } + this.context.reportSessionStart(number) + debug('session %s starting with tracks %s', number, tracks.map(track => track.name).join(', ')) + + const multivariant = Buffer.from(buildMultivariantPlaylist(tracks, known)) + if (!(await this.putChecked(connection, session, `${session.prefix}index.m3u8`, multivariant, PLAYLIST_CONTENT_TYPE))) { + return session + } + // Matter 1.6.1 11.7.1.8.2: tracks start after the multivariant playlist, each with its initial playlist and its + // CMAF header, and are uploaded in parallel + await Promise.all(tracks.map(async (track) => { + const playlist = Buffer.from(buildMediaPlaylist(track, targetDuration)) + if (!(await this.putChecked(connection, session, `${session.prefix}${track.name}/index.m3u8`, playlist, PLAYLIST_CONTENT_TYPE))) { + return + } + session.opened.add(track.id) + await this.putChecked(connection, session, `${session.prefix}${track.name}/${track.name}.init`, track.init, INIT_CONTENT_TYPE) + })) + return session + } + + private async putSegment(connection: IngestConnection, session: CMAFSession, part: CMAFTrackSegment): Promise { + const durations = session.durations.get(part.track.id)! + const path = `${session.prefix}${part.track.name}/segment_${FIRST_SEGMENT_NUMBER + durations.length}.m4s` + const headers = { + 'x-extinf-duration': formatDuration(part.duration), + } + if (await this.putChecked(connection, session, path, part.data, SEGMENT_CONTENT_TYPE, headers)) { + durations.push(part.duration) + } + } + + private async endSession(connection: IngestConnection, session: CMAFSession, tracks: CMAFTrack[], targetDuration: number): Promise { + await this.closeTracks(connection, session, tracks, targetDuration) + if (!session.failed) { + session.ended = true + this.context.reportSessionStop(session.number) + debug('session %s finished after %ds', session.number, Math.round(session.seconds)) + } + } + + private async closeTracks(connection: IngestConnection, session: CMAFSession, tracks: CMAFTrack[], targetDuration: number): Promise { + await Promise.all(tracks.filter(track => session.opened.has(track.id)).map(async (track) => { + const playlist = Buffer.from(buildMediaPlaylist(track, targetDuration, session.durations.get(track.id))) + if (await this.putChecked(connection, session, `${session.prefix}${track.name}/index.m3u8`, playlist, PLAYLIST_CONTENT_TYPE)) { + session.opened.delete(track.id) + } + })) + } + + private async putChecked(connection: IngestConnection, session: CMAFSession, path: string, body: Buffer, contentType: string, + headers: http2.OutgoingHttpHeaders = {}): Promise { + const result = await this.transfer(connection, path, body, contentType, headers) + if (result.status >= 200 && result.status < 300) { + return true + } + if (!session.failed) { + session.failed = true + const response = [result.body, result.headers].filter(Boolean).join(' ') + const detail = `PUT ${path.slice(path.indexOf('session_'))} (${body.length} bytes) -> ${result.status} ${response}` + this.context.reportError(session.number, mapStatus(result.status), detail) + } + return false + } + + // a 4xx refuses the request itself, sending it unchanged again cannot succeed; transport errors and 5xx are retried + private async transfer(connection: IngestConnection, path: string, body: Buffer, contentType: string, + headers: http2.OutgoingHttpHeaders): Promise { + for (let attempt = 0; ; attempt++) { + try { + const result = await put(connection.session(), path, body, contentType, { 'user-agent': connection.userAgent, ...headers }) + if (result.status < 500 || attempt >= MAX_RETRIES) { + return result + } + } catch (error) { + if (attempt >= MAX_RETRIES) { + throw error + } + debug('PUT %s failed, retrying: %s', path.slice(path.indexOf('session_')), error instanceof Error + ? error.message + : error) + } + connection.renew() + await sleep(RETRY_DELAY_MS) + } + } +} diff --git a/src/lib/camera/CMAFTracks.spec.ts b/src/lib/camera/CMAFTracks.spec.ts new file mode 100644 index 000000000..73fb06617 --- /dev/null +++ b/src/lib/camera/CMAFTracks.spec.ts @@ -0,0 +1,287 @@ +import { Buffer } from 'node:buffer' + +import { describe, expect, it } from 'vitest' + +import { initSegment, mediaSegment } from '../../test-utils/fmp4.js' +import { splitInitSegment, splitMediaSegment } from './CMAFTracks.js' + +const VIDEO = { id: 1, handler: 'vide' as const, timescale: 90000 } +const AUDIO = { + id: 2, + handler: 'soun' as const, + timescale: 32000, + defaultSampleDuration: 1024, +} + +function topLevel(buffer: Buffer): { type: string, start: number, end: number }[] { + const result = [] + let position = 0 + while (position + 8 <= buffer.length) { + const size = buffer.readUInt32BE(position) + result.push({ + type: buffer.toString('latin1', position + 4, position + 8), + start: position, + end: position + size, + }) + position += size + } + return result +} + +function count(buffer: Buffer, type: string): number { + let found = 0 + for (let index = buffer.indexOf(type, 0, 'latin1'); index >= 0; index = buffer.indexOf(type, index + 1, 'latin1')) { + found++ + } + return found +} + +// the payload a single-traf segment points at: moof start + trun data offset +function payload(segment: Buffer): Buffer { + const boxes = topLevel(segment) + const moof = boxes.find(entry => entry.type === 'moof')! + const mdat = boxes.find(entry => entry.type === 'mdat')! + const trun = segment.indexOf('trun', moof.start, 'latin1') - 4 + const dataOffset = segment.readInt32BE(trun + 16) + expect(moof.start + dataOffset).toBe(mdat.start + 8) + return segment.subarray(mdat.start + 8, mdat.end) +} + +describe('cMAFTracks', () => { + it('splits the init segment into one init per track', () => { + const tracks = splitInitSegment(initSegment([VIDEO, AUDIO])) + + expect(tracks).toMatchObject([ + { + id: 1, + kind: 'video', + name: 'high', + timescale: 90000, + codecs: 'hvc1.1.6.L93.B0', + width: 1920, + height: 1080, + defaultSampleDuration: 0, + }, + { + id: 2, + kind: 'audio', + name: 'audio1', + timescale: 32000, + codecs: 'mp4a.40.2', + channels: 1, + defaultSampleDuration: 1024, + }, + ]) + for (const track of tracks) { + expect(topLevel(track.init).map(entry => entry.type)).toEqual(['ftyp', 'moov']) + // CMAF headers carry the CMAF track brand + expect(track.init.toString('latin1', 16, track.init.readUInt32BE(0))).toBe('iso6cmfc') + expect(count(track.init, 'trak')).toBe(1) + expect(count(track.init, 'trex')).toBe(1) + expect(count(track.init, 'mvhd')).toBe(1) + } + expect(count(tracks[0].init, 'hvc1')).toBe(1) + expect(count(tracks[1].init, 'mp4a')).toBe(1) + }) + + it('describes an Opus audio track', () => { + const [, audio] = splitInitSegment(initSegment([ + VIDEO, + { ...AUDIO, timescale: 48000, codec: 'opus' }, + ])) + expect(audio).toMatchObject({ + kind: 'audio', + name: 'audio1', + codecs: 'Opus', + channels: 1, + }) + expect(count(audio.init, 'Opus')).toBe(1) + expect(count(audio.init, 'dOps')).toBe(1) + }) + + it('names the video tracks by resolution and numbers the audio tracks', () => { + const low = { + ...VIDEO, + id: 3, + width: 640, + height: 360, + } + const mid = { + ...VIDEO, + id: 4, + width: 1280, + height: 720, + } + const tracks = splitInitSegment(initSegment([ + low, + AUDIO, + VIDEO, + mid, + { ...AUDIO, id: 5 }, + ])) + expect(tracks.map(track => `${track.id} ${track.name}`)).toEqual([ + '1 high', + '4 mid', + '3 low', + '2 audio1', + '5 audio2', + ]) + expect(() => splitInitSegment(initSegment([ + VIDEO, + low, + mid, + { ...VIDEO, id: 6 }, + ]))).toThrow('at most 3 video tracks') + }) + + it('splits a muxed fragment and keeps every sample byte', () => { + const tracks = splitInitSegment(initSegment([VIDEO, AUDIO])) + const videoSamples = [ + Buffer.alloc(300, 0xAA), + Buffer.alloc(120, 0xAB), + ] + const audioSamples = [ + Buffer.alloc(50, 0xBA), + Buffer.alloc(60, 0xBB), + Buffer.alloc(70, 0xBC), + ] + const segment = mediaSegment(7, [ + { + trackId: 1, + durations: [180000, 180000], + samples: videoSamples, + }, + { + trackId: 2, + durations: [1024, 1024, 1024], + samples: audioSamples, + }, + ]) + + const [video, audio] = splitMediaSegment(segment, tracks) + + expect(video.track.name).toBe('high') + expect(video.duration).toBe(4) + expect(video.samples).toBe(2) + expect(topLevel(video.data).map(entry => entry.type)).toEqual(['moof', 'mdat']) + expect(payload(video.data)).toEqual(Buffer.concat(videoSamples)) + + expect(audio.track.name).toBe('audio1') + expect(audio.duration).toBeCloseTo(3072 / 32000) + expect(audio.samples).toBe(3) + expect(topLevel(audio.data).map(entry => entry.type)).toEqual(['moof', 'mdat']) + expect(payload(audio.data)).toEqual(Buffer.concat(audioSamples)) + expect(count(audio.data, 'traf')).toBe(1) + }) + + it('writes the prft again for every track', () => { + const tracks = splitInitSegment(initSegment([VIDEO, AUDIO])) + const ntp = 0xEC5F3A1000000000n + // video starts at 10 s, audio 0.1 s later; the prft refers to the video track + const segment = mediaSegment(3, [ + { + trackId: 1, + durations: [90000], + samples: [ + Buffer.alloc(40, 0x01), + ], + decodeTime: 900000n, + }, + { + trackId: 2, + durations: [1024], + samples: [ + Buffer.alloc(30, 0x02), + ], + decodeTime: 323200n, + }, + ], { + prft: { + trackId: 1, + ntp, + mediaTime: 900000n, + flags: 0x18, + }, + }) + + const [video, audio] = splitMediaSegment(segment, tracks) + for (const [part, mediaTime, expectedNtp] of [ + [video, 900000n, ntp], + [audio, 323200n, ntp + 429496729n], + ] as const) { + expect(topLevel(part.data).map(entry => entry.type)).toEqual(['prft', 'moof', 'mdat']) + expect(part.data.readUInt8(8)).toBe(1) + expect(part.data.readUInt32BE(8) & 0xFFFFFF).toBe(0x18) + expect(part.data.readUInt32BE(12)).toBe(part.track.id) + expect(part.data.readBigUInt64BE(16)).toBe(expectedNtp) + expect(part.data.readBigUInt64BE(24)).toBe(mediaTime) + expect(payload(part.data.subarray(32))).toHaveLength(part.track.kind === 'video' + ? 40 + : 30) + } + }) + + it('follows the previous traf when the moof is not the base', () => { + const tracks = splitInitSegment(initSegment([VIDEO, AUDIO])) + const videoSamples = [ + Buffer.alloc(40, 0x01), + ] + const audioSamples = [ + Buffer.alloc(30, 0x02), + ] + const segment = mediaSegment(1, [ + { + trackId: 1, + durations: [90000], + samples: videoSamples, + }, + { + trackId: 2, + durations: [1024], + samples: audioSamples, + }, + ], { defaultBaseIsMoof: false }) + + const [video, audio] = splitMediaSegment(segment, tracks) + expect(payload(video.data)).toEqual(videoSamples[0]) + expect(payload(audio.data)).toEqual(audioSamples[0]) + }) + + it('tells whether a fragment starts with a sync sample', () => { + const tracks = splitInitSegment(initSegment([VIDEO, AUDIO])) + const sample = Buffer.alloc(10) + const synced = splitMediaSegment(mediaSegment(1, [ + { + trackId: 1, + durations: [90000], + samples: [sample], + }, + ]), tracks) + const unsynced = splitMediaSegment(mediaSegment(2, [ + { + trackId: 1, + durations: [90000], + samples: [sample], + nonSyncStart: true, + }, + ]), tracks) + expect(synced[0].startsWithSync).toBe(true) + expect(unsynced[0].startsWithSync).toBe(false) + // the data offset still points at the samples after the extra first sample flags field + expect(payload(unsynced[0].data)).toEqual(sample) + }) + + it('leaves out tracks without samples in the fragment', () => { + const tracks = splitInitSegment(initSegment([VIDEO, AUDIO])) + const segment = mediaSegment(1, [ + { + trackId: 1, + durations: [90000], + samples: [ + Buffer.alloc(10), + ], + }, + ]) + expect(splitMediaSegment(segment, tracks).map(part => part.track.name)).toEqual(['high']) + }) +}) diff --git a/src/lib/camera/CMAFTracks.ts b/src/lib/camera/CMAFTracks.ts new file mode 100644 index 000000000..f1c83cf8f --- /dev/null +++ b/src/lib/camera/CMAFTracks.ts @@ -0,0 +1,614 @@ +import { Buffer } from 'node:buffer' + +/** + * One CMAF track of a muxed fMP4 recording: a CMAF track carries a single media type, so audio and video are uploaded + * as separate tracks. + * + * @group Camera Secure Video + */ +export interface CMAFTrack { + id: number + kind: 'video' | 'audio' + /** + * track name in the upload paths, the names the iCloud ingest expects: `high`, `mid` and `low` for the video tracks + * by resolution, `audio1` and up for the audio tracks + */ + name: string + timescale: number + /** RFC 6381 codecs value from the sample entry; known for HEVC, AVC and AAC */ + codecs?: string + /** video only, from the visual sample entry */ + width?: number + height?: number + /** audio only, from the audio sample entry */ + channels?: number + /** CMAF header holding only this track */ + init: Buffer + defaultSampleDuration: number + defaultSampleFlags: number +} + +/** + * The part of a media segment that belongs to one track. + * + * @group Camera Secure Video + */ +export interface CMAFTrackSegment { + track: CMAFTrack + data: Buffer + /** in seconds, from the sample durations */ + duration: number + samples: number + /** the first sample is a sync sample, so the segment can be decoded on its own */ + startsWithSync: boolean +} + +interface Box { + type: string + start: number + contentStart: number + end: number +} + +interface Run { + start: number + length: number +} + +interface ProducerReference { + flags: number + trackId: number + /** NTP timestamp, 32.32 fixed point */ + ntp: bigint + /** decode time in the timescale of the reference track */ + mediaTime: bigint +} + +const VIDEO_TRACK_NAMES = ['high', 'mid', 'low'] + +const TFHD_BASE_DATA_OFFSET = 0x1 +const TFHD_SAMPLE_DESCRIPTION_INDEX = 0x2 +const TFHD_DEFAULT_SAMPLE_DURATION = 0x8 +const TFHD_DEFAULT_SAMPLE_SIZE = 0x10 +const TFHD_DEFAULT_SAMPLE_FLAGS = 0x20 +const TFHD_DEFAULT_BASE_IS_MOOF = 0x20000 +const TRUN_DATA_OFFSET = 0x1 +const TRUN_FIRST_SAMPLE_FLAGS = 0x4 +const TRUN_SAMPLE_DURATION = 0x100 +const TRUN_SAMPLE_SIZE = 0x200 +const TRUN_SAMPLE_FLAGS = 0x400 +const TRUN_SAMPLE_COMPOSITION_TIME_OFFSET = 0x800 +const SAMPLE_IS_NON_SYNC = 0x10000 + +function boxes(buffer: Buffer, start = 0, end = buffer.length): Box[] { + const result: Box[] = [] + let position = start + while (position + 8 <= end) { + let size = buffer.readUInt32BE(position) + let contentStart = position + 8 + if (size === 1) { + size = Number(buffer.readBigUInt64BE(position + 8)) + contentStart += 8 + } else if (size === 0) { + size = end - position + } + if (size < 8 || position + size > end) { + throw new Error(`truncated ${buffer.toString('latin1', position + 4, position + 8)} box`) + } + result.push({ + type: buffer.toString('latin1', position + 4, position + 8), + start: position, + contentStart, + end: position + size, + }) + position += size + } + return result +} + +function child(buffer: Buffer, parent: Box, type: string): Box | undefined { + return boxes(buffer, parent.contentStart, parent.end).find(box => box.type === type) +} + +function slice(buffer: Buffer, box: Box): Buffer { + return buffer.subarray(box.start, box.end) +} + +function makeBox(type: string, ...children: Buffer[]): Buffer { + const header = Buffer.alloc(8) + header.writeUInt32BE(8 + children.reduce((sum, buffer) => sum + buffer.length, 0), 0) + header.write(type, 4, 'latin1') + return Buffer.concat([header, ...children]) +} + +function fullBoxVersion(buffer: Buffer, box: Box): number { + return buffer.readUInt8(box.contentStart) +} + +function trackId(buffer: Buffer, trak: Box): number { + const tkhd = child(buffer, trak, 'tkhd') + if (!tkhd) { + throw new Error('trak without tkhd') + } + return buffer.readUInt32BE(tkhd.contentStart + (fullBoxVersion(buffer, tkhd) === 1 + ? 20 + : 12)) +} + +function descriptorLength(buffer: Buffer, position: number): { length: number, next: number } { + let length = 0 + for (let i = 0; i < 4; i++) { + const byte = buffer.readUInt8(position++) + length = (length << 7) | (byte & 0x7F) + if (!(byte & 0x80)) { + break + } + } + return { length, next: position } +} + +// RFC 6381: mp4a..