Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ Run analyze, format and tests before every commit.
| `lib/src/stats/` | periodic stats report |
| `lib/src/livestream/` | WHIP/WHEP over plain HTTP, independent of the room connection |
| `lib/src/audio_output/`, `lib/src/background/` | Dart side of the SDK's native plugin: audio routing, Android foreground service, iOS CallKit |
| `lib/src/video/` | `RTCView`, `cameraDevices`, and `selectCamera`. May import `flutter_webrtc` (`test/import_boundary_test.dart`) |
| `lib/src/models/`, `wire.dart`, `logger.dart`, `version.dart` | public immutable types and shared helpers |
| `lib/src/fishjam_client.dart`, `lib/src/livestream_api.dart`, `lib/fishjam_client.dart` | the public facade; wiring and type conversion only |
| `lib/src/connectivity_network_source.dart` | the `NetworkSource` (`signaling/network_source.dart`) on `connectivity_plus`; the only file that imports it |
Expand Down
13 changes: 12 additions & 1 deletion SPEC.md
Original file line number Diff line number Diff line change
Expand Up @@ -743,7 +743,6 @@ when it joins and stops them when it leaves.
- The app adds `audio` and `voip` to `UIBackgroundModes`. Out of scope: VoIP push, incoming calls, answering,
ring timeouts.


## 20d. Picture-in-picture

`PictureInPictureView` renders a `MediaStream` and offers its first video track for picture-in-picture (PiP),
Expand Down Expand Up @@ -817,6 +816,18 @@ native plugin over the method channel `io.fishjam.client/pip` and the event chan
- When there is no video track, the app and the window both draw `fallbackLabel` on
`fallbackBackgroundColor` in `fallbackTextColor` (D24).

## 20e. Camera preview

`RTCView` renders a `MediaStream`. `mirror` (default `false`) mirrors that picture. `objectFit` is `contain` (default) or `cover`.

`cameraDevices()` lists cameras as `{deviceId, label}`, the same entries as the React Native SDK's `useCamera().cameraDevices`.

`selectCamera(deviceId, {current})` selects that camera and returns the video stream to show. [current] is the preview already showing. The same `deviceId` returns [current]. When [current] already has a video track, that track is switched in place and [current] is returned. Opening a second camera and stopping the first makes `replaceTrack` fail on Android and iOS. With no video track, `selectCamera` opens [deviceId]. An empty `deviceId` throws `ArgumentError`. This is the React Native SDK's `selectCamera(deviceId)`.

On Android, `mirror` is set on the texture renderer and takes effect on the next drawn frame. `selectCamera` pauses that drawing, switches the camera, then resumes after `RTCView` applies its current `mirror`. The frame already on screen is not flipped. Calls are `setMirror` `{textureId, mirror}` and `setPaused` `{textureId, paused}` on `io.fishjam.client/video_mirror`.

On iOS, `mirror` is a view transform. It updates when the widget rebuilds.

## 21. Logging

One SDK logger, created from `FishjamClient(debug: bool)`. Messages are prefixed with `[FISHJAM]` and emitted
Expand Down
43 changes: 43 additions & 0 deletions android/src/main/kotlin/io/fishjam/client/FishjamClientPlugin.kt
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ class FishjamClientPlugin : FlutterPlugin, ActivityAware {
private lateinit var audioOutput: AudioOutputManager
private var audioEventSink: EventChannel.EventSink? = null
private lateinit var backgroundChannel: MethodChannel
private lateinit var videoChannel: MethodChannel
private val notificationPermission = NotificationPermission()
private var activityBinding: ActivityPluginBinding? = null
private lateinit var pipChannel: MethodChannel
Expand Down Expand Up @@ -62,6 +63,8 @@ class FishjamClientPlugin : FlutterPlugin, ActivityAware {
pipEventSink = null
}
})
videoChannel = MethodChannel(binding.binaryMessenger, VIDEO_CHANNEL)
videoChannel.setMethodCallHandler(::onVideoMethodCall)
}

private fun onAudioMethodCall(call: MethodCall, result: Result) {
Expand Down Expand Up @@ -98,6 +101,44 @@ class FishjamClientPlugin : FlutterPlugin, ActivityAware {
}
}

private fun onVideoMethodCall(call: MethodCall, result: Result) {
when (call.method) {
"setMirror" -> {
val args = call.arguments as? Map<*, *>
val textureId = (args?.get("textureId") as? Number)?.toLong()
val mirror = args?.get("mirror") as? Boolean
if (textureId == null || mirror == null) {
result.error("invalid", "textureId and mirror are required", null)
return
}
try {
VideoMirror.setMirror(textureId, mirror)
result.success(null)
} catch (e: Exception) {
Log.w(TAG, "Could not set video mirror", e)
result.error("mirror", e.message, null)
}
}
"setPaused" -> {
val args = call.arguments as? Map<*, *>
val textureId = (args?.get("textureId") as? Number)?.toLong()
val paused = args?.get("paused") as? Boolean
if (textureId == null || paused == null) {
result.error("invalid", "textureId and paused are required", null)
return
}
try {
VideoMirror.setPaused(textureId, paused)
result.success(null)
} catch (e: Exception) {
Log.w(TAG, "Could not pause video", e)
result.error("paused", e.message, null)
}
}
else -> result.notImplemented()
}
}

override fun onDetachedFromEngine(binding: FlutterPlugin.FlutterPluginBinding) {
audioChannel.setMethodCallHandler(null)
audioEvents.setStreamHandler(null)
Expand All @@ -107,6 +148,7 @@ class FishjamClientPlugin : FlutterPlugin, ActivityAware {
pipChannel.setMethodCallHandler(null)
pipEvents.setStreamHandler(null)
pipEventSink = null
videoChannel.setMethodCallHandler(null)
}

override fun onAttachedToActivity(binding: ActivityPluginBinding) {
Expand Down Expand Up @@ -160,5 +202,6 @@ class FishjamClientPlugin : FlutterPlugin, ActivityAware {
private const val BACKGROUND_CHANNEL = "io.fishjam.client/background"
private const val PIP_CHANNEL = "io.fishjam.client/pip"
private const val PIP_EVENTS_CHANNEL = "io.fishjam.client/pip_events"
private const val VIDEO_CHANNEL = "io.fishjam.client/video_mirror"
}
}
50 changes: 50 additions & 0 deletions android/src/main/kotlin/io/fishjam/client/VideoMirror.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
package io.fishjam.client

import android.util.LongSparseArray

/**
* Mirror and pause for flutter_webrtc's texture renderer, looked up by
* reflection so this plugin does not depend on flutter_webrtc.
* `com.cloudwebrtc.webrtc.**` is kept by that package, so these names
* survive shrinking.
*
* [setPaused] `true` calls `pauseVideo` (the last frame stays). `false`
* calls `disableFpsReduction`. The mirror is set while drawing is paused.
*/
internal object VideoMirror {
fun setMirror(textureId: Long, mirror: Boolean) {
invoke(textureId, "setMirror", mirror)
}

fun setPaused(textureId: Long, paused: Boolean) {
invoke(textureId, if (paused) "pauseVideo" else "disableFpsReduction")
}

private fun invoke(textureId: Long, methodName: String, argument: Boolean? = null) {
val renderer = renderer(textureId)
val method = renderer.javaClass.methods.firstOrNull { method ->
method.name == methodName &&
method.parameterTypes.size == if (argument == null) 0 else 1
} ?: error("video renderer has no $methodName")
if (argument == null) method.invoke(renderer) else method.invoke(renderer, argument)
}

private fun renderer(textureId: Long): Any {
val pluginClass = Class.forName("com.cloudwebrtc.webrtc.FlutterWebRTCPlugin")
val plugin = pluginClass.getField("sharedSingleton").get(null)
?: error("FlutterWebRTCPlugin.sharedSingleton is null")
val handler = readField(plugin, "methodCallHandler")
?: error("flutter_webrtc has no method handler")
val renders = readField(handler, "renders") as LongSparseArray<*>
val render = renders.get(textureId)
?: error("no video renderer for texture $textureId")
return readField(render, "surfaceTextureRenderer")
?: error("video renderer has no surface")
}

private fun readField(target: Any, name: String): Any? {
val field = target.javaClass.getDeclaredField(name)
field.isAccessible = true
return field.get(target)
}
}
1 change: 0 additions & 1 deletion example/integration_test/room_test.dart
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import 'dart:typed_data';

import 'package:fishjam_client/fishjam_client.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:flutter_webrtc/flutter_webrtc.dart';
import 'package:http/http.dart' as http;
import 'package:integration_test/integration_test.dart';

Expand Down
98 changes: 65 additions & 33 deletions example/lib/room_screen.dart
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,11 @@ import 'dart:io';

import 'package:fishjam_client/fishjam_client.dart';
import 'package:flutter/material.dart';
import 'package:flutter_webrtc/flutter_webrtc.dart';

import 'sandbox.dart';

/// A minimal room: publishes camera and microphone, renders remote video,
/// mute toggles and leave.
/// mute toggles, camera switch, picture-in-picture and leave.
class RoomScreen extends StatefulWidget {
const RoomScreen({
super.key,
Expand Down Expand Up @@ -53,12 +52,14 @@ class _RoomScreenState extends State<RoomScreen> {
final _subscriptions = <StreamSubscription<Object?>>[];

MediaStream? _localStream;
MediaStream? _cameraStream;
MediaStreamTrack? _camera;
MediaStreamTrack? _microphone;
String? _cameraTrackId;
String? _microphoneTrackId;
bool _cameraOn = true;
bool _microphoneOn = true;
bool _frontCamera = true;
String _status = 'Connecting…';
final _audioOutput = AudioOutputManager.instance;
final _callKit = CallKit.instance;
Expand Down Expand Up @@ -195,6 +196,7 @@ class _RoomScreenState extends State<RoomScreen> {
'video': {'facingMode': 'user'},
});
_localStream = stream;
_cameraStream = stream;
_camera = stream.getVideoTracks().firstOrNull;
_microphone = stream.getAudioTracks().firstOrNull;
if (mounted) setState(() {});
Expand Down Expand Up @@ -325,6 +327,35 @@ class _RoomScreenState extends State<RoomScreen> {
);
}

Future<void> _switchCamera() async {
final camera = _camera;
if (camera == null) return;
try {
final cameras = await cameraDevices();
final currentId = camera.getSettings()['deviceId'];
final next = cameras
.where((device) => device.deviceId != currentId)
.firstOrNull;
if (next == null) return;
final stream = await selectCamera(next.deviceId, current: _cameraStream);
final track = stream.getVideoTracks().firstOrNull;
if (track != null && _cameraTrackId != null && track.id != camera.id) {
await _client.replaceTrack(_cameraTrackId!, track);
}
final label = next.label.toLowerCase();
final front = label.contains('front');
final back = label.contains('back');
if (!mounted) return;
setState(() {
_cameraStream = stream;
_camera = track;
if (front || back) _frontCamera = front;
});
} on Object catch (e) {
_setStatus('Error: $e');
}
}

Future<void> _toggleMicrophone() async {
final on = !_microphoneOn;
setState(() => _microphoneOn = on);
Expand Down Expand Up @@ -385,27 +416,40 @@ class _RoomScreenState extends State<RoomScreen> {
}
unawaited(stream.dispose());
}
final cameraStream = _cameraStream;
if (cameraStream != null && cameraStream != stream) {
for (final track in cameraStream.getTracks()) {
unawaited(track.stop());
}
unawaited(cameraStream.dispose());
}
super.dispose();
}

@override
Widget build(BuildContext context) {
final tiles = [
_VideoTile(
stream: _localStream,
label: '${widget.displayName} (you)',
mirror: true,
pip: _localPip,
pipOptions: _noPip,
child: RTCView(
mediaStream: _cameraStream,
mirror: _frontCamera,
objectFit: RTCViewObjectFit.cover,
),
),
for (final video in _remote.values)
for (final entry in _remote.entries)
_VideoTile(
stream: video.stream,
label: video.label,
pip: identical(video, _remote.values.first) ? _remotePip : null,
pipOptions: identical(video, _remote.values.first)
? _pipOptions
: _noPip,
key: ValueKey(entry.key),
label: entry.value.label,
child: PictureInPictureView(
mediaStream: entry.value.stream,
controller: identical(entry.value, _remote.values.first)
? _remotePip
: null,
pipOptions: identical(entry.value, _remote.values.first)
? _pipOptions
: _noPip,
),
),
];
return Scaffold(
Expand Down Expand Up @@ -443,6 +487,11 @@ class _RoomScreenState extends State<RoomScreen> {
onPressed: _toggleCamera,
icon: Icon(_cameraOn ? Icons.videocam : Icons.videocam_off),
),
IconButton.filledTonal(
tooltip: 'Switch camera',
onPressed: _camera == null ? null : _switchCamera,
icon: const Icon(Icons.cameraswitch),
),
IconButton.filledTonal(
tooltip: 'Picture in picture',
onPressed:
Expand All @@ -468,35 +517,18 @@ class _RoomScreenState extends State<RoomScreen> {
}

class _VideoTile extends StatelessWidget {
const _VideoTile({
required this.label,
required this.pipOptions,
this.stream,
this.pip,
this.mirror = false,
});
const _VideoTile({super.key, required this.child, required this.label});

final MediaStream? stream;
final Widget child;
final String label;
final PictureInPictureController? pip;
final PictureInPictureOptions pipOptions;
final bool mirror;

@override
Widget build(BuildContext context) => ClipRRect(
borderRadius: BorderRadius.circular(8),
child: Stack(
fit: StackFit.expand,
children: [
ColoredBox(
color: Colors.black,
child: PictureInPictureView(
mediaStream: stream,
controller: pip,
mirror: mirror,
pipOptions: pipOptions,
),
),
ColoredBox(color: Colors.black, child: child),
Positioned(
left: 8,
bottom: 8,
Expand Down
5 changes: 5 additions & 0 deletions lib/fishjam_client.dart
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
/// Fishjam client SDK for Flutter. Start with `FishjamClient`.
library;

export 'package:flutter_webrtc/flutter_webrtc.dart'
show MediaStream, MediaStreamTrack, navigator;

export 'src/audio_output/audio_output_manager.dart'
show
AndroidAudioOutput,
Expand Down Expand Up @@ -61,3 +64,5 @@ export 'src/rtc/rtc_adapter.dart' show MediaKind, PeerConnectionState, RtcStats;
export 'src/signaling/auth.dart' show fishjamUrlFromId;
export 'src/tracks/bandwidth.dart' show simulcastCapsKbps, singleStreamCapKbps;
export 'src/version.dart' show packageVersion;
export 'src/video/rtc_view.dart'
show CameraDevice, RTCView, RTCViewObjectFit, cameraDevices, selectCamera;
Loading
Loading