From 550b38224b4f49cf63af1d4ba525287cb8781f1d Mon Sep 17 00:00:00 2001 From: Joao Santos Date: Thu, 17 Apr 2025 00:07:54 -0300 Subject: [PATCH 1/2] emit server device updates --- packages/webrtc/src/BaseConnection.ts | 30 ++++------------ packages/webrtc/src/utils/helpers.ts | 36 ++++++++++++++++++- packages/webrtc/src/utils/interfaces.ts | 10 +++++- .../webrtc/src/workers/vertoEventWorker.ts | 24 ++++++++++--- 4 files changed, 71 insertions(+), 29 deletions(-) diff --git a/packages/webrtc/src/BaseConnection.ts b/packages/webrtc/src/BaseConnection.ts index e7f9bfb85..91147cbaf 100644 --- a/packages/webrtc/src/BaseConnection.ts +++ b/packages/webrtc/src/BaseConnection.ts @@ -43,6 +43,7 @@ import { INVITE_VERSION, VIDEO_CONSTRAINTS, } from './utils/constants' +import { emitDeviceUpdatedEventHelper } from './utils/helpers' export type BaseConnectionOptions = ConnectionOptions & BaseComponentOptions @@ -693,29 +694,12 @@ export class BaseConnection< prevAudioTrack, prevVideoTrack, }: EmitDeviceUpdatedEventsParams) { - if (newTrack.kind === 'audio') { - this.emit('microphone.updated', { - previous: { - deviceId: prevAudioTrack?.id, - label: prevAudioTrack?.label, - }, - current: { - deviceId: newTrack.id, - label: newTrack.label, - }, - }) - } else if (newTrack.kind === 'video') { - this.emit('camera.updated', { - previous: { - deviceId: prevVideoTrack?.id, - label: prevVideoTrack?.label, - }, - current: { - deviceId: newTrack.id, - label: newTrack.label, - }, - }) - } + emitDeviceUpdatedEventHelper({ + prevAudioTrack, + prevVideoTrack, + newTrack, + emitFn: this.emit.bind(this), + }) } /** diff --git a/packages/webrtc/src/utils/helpers.ts b/packages/webrtc/src/utils/helpers.ts index cbf0b38cc..597f5cf95 100644 --- a/packages/webrtc/src/utils/helpers.ts +++ b/packages/webrtc/src/utils/helpers.ts @@ -1,7 +1,10 @@ import { getLogger } from '@signalwire/core' import { getUserMedia as _getUserMedia } from './getUserMedia' import { assureDeviceId } from './deviceHelpers' -import { ConnectionOptions } from './interfaces' +import { + ConnectionOptions, + EmitDeviceUpdatedEventHelperParams, +} from './interfaces' import { sdpHasAudio, sdpHasVideo } from './sdpHelpers' // FIXME: Remove and use getUserMedia directly @@ -100,3 +103,34 @@ export const filterIceServers = ( urls: disableUdpIceServers ? filterOutUdpUrls(server.urls) : server.urls, })) } + +export const emitDeviceUpdatedEventHelper = ({ + prevAudioTrack, + prevVideoTrack, + newTrack, + emitFn, +}: EmitDeviceUpdatedEventHelperParams) => { + if (newTrack.kind === 'audio') { + emitFn('microphone.updated', { + previous: { + deviceId: prevAudioTrack?.id, + label: prevAudioTrack?.label, + }, + current: { + deviceId: newTrack.id, + label: newTrack.label, + }, + }) + } else if (newTrack.kind === 'video') { + emitFn('camera.updated', { + previous: { + deviceId: prevVideoTrack?.id, + label: prevVideoTrack?.label, + }, + current: { + deviceId: newTrack.id, + label: newTrack.label, + }, + }) + } +} diff --git a/packages/webrtc/src/utils/interfaces.ts b/packages/webrtc/src/utils/interfaces.ts index bda64f22d..fdabf9311 100644 --- a/packages/webrtc/src/utils/interfaces.ts +++ b/packages/webrtc/src/utils/interfaces.ts @@ -1,4 +1,4 @@ -import type { VideoPositions } from '@signalwire/core' +import type { EventEmitter, VideoPositions } from '@signalwire/core' import { BaseConnectionState, VideoRoomDeviceEventParams, @@ -108,6 +108,14 @@ export interface EmitDeviceUpdatedEventsParams { prevVideoTrack?: MediaStreamTrack | null } +export interface EmitDeviceUpdatedEventHelperParams + extends EmitDeviceUpdatedEventsParams { + emitFn: >( + event: E, + ...args: EventEmitter.EventArgs + ) => boolean +} + export type UpdateMediaOptionsParams = Pick< ConnectionOptions, 'video' | 'audio' | 'negotiateVideo' | 'negotiateAudio' diff --git a/packages/webrtc/src/workers/vertoEventWorker.ts b/packages/webrtc/src/workers/vertoEventWorker.ts index 05eef564f..419334e92 100644 --- a/packages/webrtc/src/workers/vertoEventWorker.ts +++ b/packages/webrtc/src/workers/vertoEventWorker.ts @@ -15,6 +15,7 @@ import { } from '@signalwire/core' import { BaseConnection } from '../BaseConnection' +import { emitDeviceUpdatedEventHelper } from '../utils/helpers' type VertoEventWorkerOnDone = (args: BaseConnection) => void type VertoEventWorkerOnFail = (args: { error: Error }) => void @@ -129,11 +130,28 @@ export const vertoEventWorker: SDKWorker< break } const { audio, video } = params.mediaParams + + const prevAudioTrack = peer?.localAudioTrack?.clone() + const prevVideoTrack = peer?.localVideoTrack?.clone() if (peer && video) { - peer.applyMediaConstraints('video', video) + peer.applyMediaConstraints('video', video).then(() => { + emitDeviceUpdatedEventHelper({ + prevAudioTrack, + prevVideoTrack, + newTrack: peer?.localVideoTrack!, + emitFn: instance.emit, + }) + }) } if (peer && audio) { - peer.applyMediaConstraints('audio', audio) + peer.applyMediaConstraints('audio', audio).then(() => { + emitDeviceUpdatedEventHelper({ + prevAudioTrack, + prevVideoTrack, + newTrack: peer?.localAudioTrack!, + emitFn: instance.emit, + }) + }) } break } @@ -164,6 +182,4 @@ export const vertoEventWorker: SDKWorker< }) yield sagaEffects.fork(catchableWorker, action) } - - getLogger().trace('vertoEventWorker ended') } From ddb9c67209e46ae49db40ce7498f1d9abbc1ad69 Mon Sep 17 00:00:00 2001 From: Joao Santos Date: Thu, 17 Apr 2025 18:39:23 -0300 Subject: [PATCH 2/2] new constraints event --- packages/core/src/types/videoRoomDevice.ts | 22 ++++- packages/webrtc/src/BaseConnection.ts | 84 ++++++++++++++++--- packages/webrtc/src/RTCPeer.ts | 41 +++++---- packages/webrtc/src/utils/helpers.ts | 36 +------- packages/webrtc/src/utils/interfaces.ts | 11 ++- .../webrtc/src/workers/vertoEventWorker.ts | 57 +++++++++---- 6 files changed, 171 insertions(+), 80 deletions(-) diff --git a/packages/core/src/types/videoRoomDevice.ts b/packages/core/src/types/videoRoomDevice.ts index d9d50d42a..02b0e240f 100644 --- a/packages/core/src/types/videoRoomDevice.ts +++ b/packages/core/src/types/videoRoomDevice.ts @@ -1,6 +1,8 @@ export type CameraUpdated = 'camera.updated' +export type CameraConstraintsUpdated = 'camera.constraints.updated' export type CameraDisconnected = 'camera.disconnected' export type MicrophoneUpdated = 'microphone.updated' +export type MicrophoneConstraintsUpdated = 'microphone.constraints.updated' export type MicrophoneDisconnected = 'microphone.disconnected' export type SpeakerUpdated = 'speaker.updated' export type SpeakerDisconnected = 'speaker.disconnected' @@ -14,6 +16,10 @@ export type VideoRoomDeviceUpdatedEventNames = | MicrophoneUpdated | SpeakerUpdated +export type VideoRoomDeviceConstraintsUpdatedEventNames = + | CameraConstraintsUpdated + | MicrophoneConstraintsUpdated + export type VideoRoomDeviceDisconnectedEventNames = | CameraDisconnected | MicrophoneDisconnected @@ -21,10 +27,11 @@ export type VideoRoomDeviceDisconnectedEventNames = export type VideoRoomDeviceEventNames = | VideoRoomDeviceUpdatedEventNames + | VideoRoomDeviceConstraintsUpdatedEventNames | VideoRoomDeviceDisconnectedEventNames export interface VideoRoomMediaDeviceInfo { - deviceId: MediaDeviceInfo['deviceId'] | undefined + deviceId: MediaTrackConstraintSet['deviceId'] label: MediaDeviceInfo['label'] | undefined } @@ -33,8 +40,21 @@ export interface DeviceUpdatedEventParams { current: VideoRoomMediaDeviceInfo } +export interface MediaDeviceIdentifiers { + deviceId: MediaTrackConstraintSet['deviceId'] + kind: string + label: string +} +export interface DeviceConstraintsUpdatedEventParams { + kind: string + previous: MediaDeviceIdentifiers | undefined + current: MediaDeviceIdentifiers + constraints: MediaTrackConstraints +} + export type DeviceDisconnectedEventParams = VideoRoomMediaDeviceInfo export type VideoRoomDeviceEventParams = | DeviceUpdatedEventParams | DeviceDisconnectedEventParams + | DeviceConstraintsUpdatedEventParams diff --git a/packages/webrtc/src/BaseConnection.ts b/packages/webrtc/src/BaseConnection.ts index 91147cbaf..a0693b6ef 100644 --- a/packages/webrtc/src/BaseConnection.ts +++ b/packages/webrtc/src/BaseConnection.ts @@ -21,7 +21,11 @@ import { UpdateMediaParams, UpdateMediaDirection, } from '@signalwire/core' -import type { ReduxComponent, VertoModifyResponse } from '@signalwire/core' +import type { + ReduxComponent, + VertoModifyResponse, + VideoRoomDeviceConstraintsUpdatedEventNames, +} from '@signalwire/core' import RTCPeer from './RTCPeer' import { ConnectionOptions, @@ -29,6 +33,7 @@ import { UpdateMediaOptionsParams, BaseConnectionEvents, OnVertoByeParams, + EmitDeviceConstraintsUpdatedEventsParams, } from './utils/interfaces' import { stopTrack, getUserMedia, streamIsValid } from './utils' import { @@ -43,7 +48,6 @@ import { INVITE_VERSION, VIDEO_CONSTRAINTS, } from './utils/constants' -import { emitDeviceUpdatedEventHelper } from './utils/helpers' export type BaseConnectionOptions = ConnectionOptions & BaseComponentOptions @@ -564,11 +568,30 @@ export class BaseConnection< // Add or update the transceiver (may trigger renegotiation) await this.handleTransceiverForTrack(newTrack) - // Emit the device.updated events - this.emitDeviceUpdatedEvents({ - newTrack, - prevAudioTrack, - prevVideoTrack, + const prevTrack = newTrack.kind === 'audio' ? + prevAudioTrack: prevVideoTrack + + if (newTrack.getConstraints().deviceId !== prevTrack?.getConstraints().deviceId) { + // Emit the device.updated events only when the device was updated + this.emitDeviceUpdatedEvents({ + newTrack, + prevAudioTrack, + prevVideoTrack, + }) + } + + this.emitDeviceConstraintsUpdatedEvents({ + currentConstraints: newTrack.getConstraints(), + prevTrackIdentifiers: prevTrack ? { + kind: prevTrack?.kind, + label: prevTrack?.label, + deviceId: prevTrack?.getConstraints().deviceId + } : undefined, + currentTrackIdentifiers: { + kind: newTrack.kind, + label: newTrack.label, + deviceId: newTrack.getConstraints().deviceId + } }) } @@ -694,11 +717,48 @@ export class BaseConnection< prevAudioTrack, prevVideoTrack, }: EmitDeviceUpdatedEventsParams) { - emitDeviceUpdatedEventHelper({ - prevAudioTrack, - prevVideoTrack, - newTrack, - emitFn: this.emit.bind(this), + if (newTrack.kind === 'audio') { + this.emit('microphone.updated', { + previous: { + // https://developer.mozilla.org/en-US/docs/Web/API/MediaStreamTrack/id + deviceId: prevAudioTrack?.getConstraints().deviceId, + label: prevAudioTrack?.label, + }, + current: { + deviceId: newTrack.getConstraints().deviceId, + label: newTrack.label, + }, + }) + } else if (newTrack.kind === 'video') { + this.emit('camera.updated', { + previous: { + deviceId: prevVideoTrack?.getConstraints().deviceId, + label: prevVideoTrack?.label, + }, + current: { + deviceId: newTrack.getConstraints().deviceId, + label: newTrack.label, + }, + }) + } + } + + //** internal */ + public emitDeviceConstraintsUpdatedEvents({ + currentConstraints, + prevTrackIdentifiers, + currentTrackIdentifiers, + }: EmitDeviceConstraintsUpdatedEventsParams) { + + const eventPrefix = + currentTrackIdentifiers.kind === 'audioinput' ? 'microphone' : 'camera' + const event: VideoRoomDeviceConstraintsUpdatedEventNames = `${eventPrefix}.constraints.updated` + + this.emit(event, { + kind: currentTrackIdentifiers.kind, + previous: prevTrackIdentifiers, + current: currentTrackIdentifiers, + constraints: currentConstraints, }) } diff --git a/packages/webrtc/src/RTCPeer.ts b/packages/webrtc/src/RTCPeer.ts index c2faab702..6153e231d 100644 --- a/packages/webrtc/src/RTCPeer.ts +++ b/packages/webrtc/src/RTCPeer.ts @@ -368,30 +368,39 @@ export default class RTCPeer { try { const sender = this._getSenderByKind(kind) if (!sender || !sender.track) { - return this.logger.info( - 'No sender to apply constraints', + // TODO this is breaking chance we need to eval if we want to have it on VideoSDK + this.logger.error( + 'No sender track to apply constraints', kind, constraints ) + throw new Error('No sender track to apply constraints') } - if (sender.track.readyState === 'live') { - const newConstraints: MediaTrackConstraints = { - ...sender.track.getConstraints(), - ...constraints, - } - const deviceId = this.getDeviceId(kind) - if (deviceId && !this.options.screenShare) { - newConstraints.deviceId = { exact: deviceId } - } - this.logger.info( - `Apply ${kind} constraints`, - this.call.id, - newConstraints + if (sender.track.readyState !== 'live') { + this.logger.error( + 'Sender track is not live to apply constraints', + kind, + constraints ) - await sender.track.applyConstraints(newConstraints) + throw new Error('Sender track is not live to apply constraints') } + const newConstraints: MediaTrackConstraints = { + ...sender.track.getConstraints(), + ...constraints, + } + const deviceId = this.getDeviceId(kind) + if (deviceId && !this.options.screenShare) { + newConstraints.deviceId = { exact: deviceId } + } + this.logger.info( + `Apply ${kind} constraints`, + this.call.id, + newConstraints + ) + await sender.track.applyConstraints(newConstraints) } catch (error) { this.logger.error('Error applying constraints', kind, constraints) + throw error } } diff --git a/packages/webrtc/src/utils/helpers.ts b/packages/webrtc/src/utils/helpers.ts index 597f5cf95..cbf0b38cc 100644 --- a/packages/webrtc/src/utils/helpers.ts +++ b/packages/webrtc/src/utils/helpers.ts @@ -1,10 +1,7 @@ import { getLogger } from '@signalwire/core' import { getUserMedia as _getUserMedia } from './getUserMedia' import { assureDeviceId } from './deviceHelpers' -import { - ConnectionOptions, - EmitDeviceUpdatedEventHelperParams, -} from './interfaces' +import { ConnectionOptions } from './interfaces' import { sdpHasAudio, sdpHasVideo } from './sdpHelpers' // FIXME: Remove and use getUserMedia directly @@ -103,34 +100,3 @@ export const filterIceServers = ( urls: disableUdpIceServers ? filterOutUdpUrls(server.urls) : server.urls, })) } - -export const emitDeviceUpdatedEventHelper = ({ - prevAudioTrack, - prevVideoTrack, - newTrack, - emitFn, -}: EmitDeviceUpdatedEventHelperParams) => { - if (newTrack.kind === 'audio') { - emitFn('microphone.updated', { - previous: { - deviceId: prevAudioTrack?.id, - label: prevAudioTrack?.label, - }, - current: { - deviceId: newTrack.id, - label: newTrack.label, - }, - }) - } else if (newTrack.kind === 'video') { - emitFn('camera.updated', { - previous: { - deviceId: prevVideoTrack?.id, - label: prevVideoTrack?.label, - }, - current: { - deviceId: newTrack.id, - label: newTrack.label, - }, - }) - } -} diff --git a/packages/webrtc/src/utils/interfaces.ts b/packages/webrtc/src/utils/interfaces.ts index fdabf9311..e94516593 100644 --- a/packages/webrtc/src/utils/interfaces.ts +++ b/packages/webrtc/src/utils/interfaces.ts @@ -1,4 +1,8 @@ -import type { EventEmitter, VideoPositions } from '@signalwire/core' +import type { + EventEmitter, + MediaDeviceIdentifiers, + VideoPositions, +} from '@signalwire/core' import { BaseConnectionState, VideoRoomDeviceEventParams, @@ -107,6 +111,11 @@ export interface EmitDeviceUpdatedEventsParams { prevAudioTrack?: MediaStreamTrack | null prevVideoTrack?: MediaStreamTrack | null } +export interface EmitDeviceConstraintsUpdatedEventsParams { + currentConstraints: MediaTrackConstraints + prevTrackIdentifiers: MediaDeviceIdentifiers | undefined + currentTrackIdentifiers: MediaDeviceIdentifiers +} export interface EmitDeviceUpdatedEventHelperParams extends EmitDeviceUpdatedEventsParams { diff --git a/packages/webrtc/src/workers/vertoEventWorker.ts b/packages/webrtc/src/workers/vertoEventWorker.ts index 419334e92..f419d1747 100644 --- a/packages/webrtc/src/workers/vertoEventWorker.ts +++ b/packages/webrtc/src/workers/vertoEventWorker.ts @@ -132,26 +132,53 @@ export const vertoEventWorker: SDKWorker< const { audio, video } = params.mediaParams const prevAudioTrack = peer?.localAudioTrack?.clone() - const prevVideoTrack = peer?.localVideoTrack?.clone() if (peer && video) { - peer.applyMediaConstraints('video', video).then(() => { - emitDeviceUpdatedEventHelper({ - prevAudioTrack, - prevVideoTrack, - newTrack: peer?.localVideoTrack!, - emitFn: instance.emit, + const prevTrackIdentifiers = peer?.localVideoTrack + ? { + kind: peer.localVideoTrack.kind, + label: peer.localVideoTrack.label, + deviceId: peer.localVideoTrack.getConstraints().deviceId, + } + : undefined + + peer + .applyMediaConstraints('video', video) + .then(() => { + instance.emitDeviceConstraintsUpdatedEvents({ + currentConstraints: peer.localVideoTrack!.getConstraints(), + prevTrackIdentifiers, + currentTrackIdentifiers: { + kind: peer.localVideoTrack!.kind, + label: peer.localVideoTrack!.label, + deviceId: peer.localVideoTrack!.getConstraints().deviceId, + }, + }) }) - }) + .catch(console.error) } if (peer && audio) { - peer.applyMediaConstraints('audio', audio).then(() => { - emitDeviceUpdatedEventHelper({ - prevAudioTrack, - prevVideoTrack, - newTrack: peer?.localAudioTrack!, - emitFn: instance.emit, + const prevTrackIdentifiers = peer?.localAudioTrack + ? { + kind: peer.localAudioTrack.kind, + label: peer.localAudioTrack.label, + deviceId: peer.localAudioTrack.getConstraints().deviceId, + } + : undefined + + peer + .applyMediaConstraints('audio', audio) + .then(() => { + instance.emitDeviceConstraintsUpdatedEvents({ + currentConstraints: peer.localAudioTrack!.getConstraints(), + prevTrackIdentifiers, + currentTrackIdentifiers: { + kind: peer.localAudioTrack!.kind, + label: peer.localAudioTrack!.label, + deviceId: peer.localAudioTrack!.getConstraints().deviceId, + }, + }) }) - }) + .catch(console.error) } break }