0

I am trying to connect a Kurento Media Server to React Native on iOS for a group video call. The server is running on Nodejs and Docker, and they already have a TURN server set up.

The app is already running in the web browser via kurento-utils, but I can't stream the web video to RN, instead, the RN video looks just fine on the web.

I'm not usig kurento-utils on client side of React Native.

This is my code:

Video Screen (emmiter and receptor)

import React, { useState, useEffect } from 'react';
import { 
    View,
    StyleSheet,
    Text,
} from 'react-native';

import Display from 'react-native-display';
import InCallManager from 'react-native-incall-manager';
import { TouchableOpacity } from 'react-native-gesture-handler';



import { socket } from '../../screens/User';
import ReceiveScreen from './ReceiveScreen';
import {
    RTCView,
    RTCIceCandidate,
} from 'react-native-webrtc';
import { 
    startCommunication, 
    receiveVideo,
    addIceCandidate, 
    ProcessAnswer,
    ViewPCArray
} from '../../utils/webrtc-utils';
const participants = {};

function sendMessage(message) {
    if (socket) {
        socket.emit('message', message);
    }
}

const VideoScreen = () => {
        const [videoURL, setVideoURL] = useState(null)
        const [remoteURL, setRemoteURL] = useState(null)
        const [userName, setUserName] = useState(socket.id)
        const [roomName, setRoomName] = useState('9b33737f-737f-4a3d-a323-e1cd3f4b68b2')


    useEffect(() => {
        if(socket) {
            var message = {
                event: 'joinRoom',
                userName: userName,
                roomName: roomName
            }
            sendMessage(message);
        }

        socket.on('message', message => {
            messageProcessHandler(message);
        });

        return () => {
            socket.close();
        }
    }, [])

    /**
     * 
     * @param {*} msg 
     */
    const messageProcessHandler = (msg) => {
        // console.log(`MessageProcessHandler: ${msg.event}`);
        switch (msg.event) {
            case 'existingParticipants':
                console.log('LOG:176 onExistingParticipants= ' + JSON.stringify(msg));
                startCommunication(sendMessage, userName, (stream) => {
                    setVideoURL(stream.toURL())
                    msg.existingUsers.forEach((object) => {
                        participants[object.name] = object.id;
                        console.log("participants:" + JSON.stringify(participants));
                        receiveVideo(sendMessage, object.name, (pc) => {
                            console.log('getRemoteStreams ', pc);
                            pc.getRemoteStreams().forEach(track => {
                                // console.log('TRACK: ', track);
                                console.log('STREAM', track);
                                // pc.addTrack(track, stream);
                                setRemoteURL(track.toURL())
                            })
                        });
                    });
                });
                break;
            case 'newParticipantArrived':
                participants[msg.name] = msg.name;            
                if (remoteURL == null || remoteURL === '') {
                    receiveVideo(sendMessage, msg.userName, (pc) => {   
                        pc.getRemoteStreams().forEach(track => {
                            console.log('STREAM', track);
                            setRemoteURL(track.toURL())
                        })
                    });
                }
                break;
            case 'participantLeft':
                participantLeft(msg.name);   
                break;
            case 'receiveVideoAnswer':
                ProcessAnswer(msg.senderid, msg.sdpAnswer, (err) => {
                    if (err) {
                        console.log('the error: ' + err);
                    }
                });
                break;
            case 'candidate':
                addIceCandidate(msg.userid, new RTCIceCandidate(msg.candidate));
                break;
            default:
                console.error('Unrecognized message', msg.message);
        }
    }

    /**
     * 
     * @param {*} name 
     */
    const participantLeft = (name) => {
        if (participants[name]) {
            delete participants[name];
        }

        if (Object.keys(participants).length == 0) {
            setRemoteURL(null)
        }
    }

    return (
        <View style={styles.container}>
            <View style={{position: "absolute", top: 40, right: 20, zIndex: 200}}>
                <Text style={{fontSize: 20, fontWeight: "bold", color: "#fff"}}>{userName}</Text>
            </View>
            <RTCView zOrder={0} objectFit='cover' style={styles.videoContainer} streamURL={videoURL}  />

            <Display enable={remoteURL != null}>
                <View style={styles.floatView}>
                    <ReceiveScreen videoURL={remoteURL} />
                </View>
            </Display>
        </View>
    );

}

const styles = StyleSheet.create({
    container: {
        flex: 1
    },
    videoContainer: {
        flex: 1
    },
    floatView: {
        position: 'absolute',
        width: 250,
        height: 210,
        bottom: 15,
        right: 20,
        backgroundColor: 'rgba(255, 255, 255, 0.1)',
        borderRadius: 15
    }
});
export default VideoScreen

Display (where the browser video is supposed to be)

import React, { Component } from 'react';
import config from "../../config/app.js";
import { 
    View,
    Text,
    ScrollView,
    StyleSheet
} from 'react-native';

import { RTCView } from 'react-native-webrtc';

const styles = StyleSheet.create({
    root: {
        flex: 1
    },
    titleContainer: {
        flex: 0.1,
        paddingVertical: 2,
        paddingLeft: 10
    },
    title: {
        fontSize: 15,
        color: 'rgba(255, 255, 255, 0.5)'
    },
    contentContainer: {
       flex: 1,
       backgroundColor: 'transparent'
    }
});


const ReceiveScreen = (props) => {
    return (
        <View style={styles.root}>
            <View style={styles.titleContainer}>
                <Text style={styles.title}>Nuevo miembro</Text>
            </View>
            <RTCView objectFit='cover' zOrder={1} style={styles.contentContainer} streamURL={props.videoURL} mirror={true} />
        </View>
    )
}

export default ReceiveScreen;

WebRtc Utils

import {
    getUserMedia,
    MediaStreamTrack,
    mediaDevices,
    RTCPeerConnection,
    RTCSessionDescription,
} from 'react-native-webrtc';

import { PERMISSIONS, request, checkMultiple } from 'react-native-permissions';

import { socket } from '../screens/User';

let pcArray = {};
let isFront = true;
let isEnableAudio = true;
let isEnableVideo = true;
let localstream = null;
const ICE_CONFIG = {
    'iceServers': [
        {
            'urls': 'turn:xxx.xxx.xxx.xxx:3478',
            'username': 'xxxxxxxx',
            'credential': 'xxxxxxxxxxxxxxxxxxxxxxx',
            'credentialType': 'password'
        }
    ]
};

/**
 * Obtenga elementos multimedia locales (transmisiones de video)
 *
 * @param {*} _sendMessage
 * @param {*} _name
 * @param {*} callback
 */
export function startCommunication(_sendMessage, _name, callback) {
    getStream(true, stream => {
        localstream = stream;
        let options = {
            audio: true,
            video : {
                mandatory : {
                    maxWidth : 320,
                    maxFrameRate : 15,
                    minFrameRate : 1
                }
            }
        };
        let pc = createPC(_sendMessage, _name, true, options);
        pcArray[_name] = pc;
        callback(stream);
    });
}
/**
 * Obtener transmisión de video remota
 *
 * @param {*} _sendMessae
 * @param {*} _name
 * @param {*} callback
 */
export function receiveVideo(_sendMessae, _name, callback) {
    console.log("receiveVideo:", _sendMessae, _name, callback);
    let options = {
        audio: true,
        video : {
            mandatory : {
                // maxWidth : 320,
                // maxFrameRate : 15,
                // minFrameRate : 15
                maxWidth : 120,
                maxHeight: 80,
                minWidth : 80,
                minHeight: 60,
                maxFrameRate : 10,
                minFrameRate : 10
            }
        }
    };
    let pc = createPC(_sendMessae, _name, true, options);
    // console.log(`PC CREATED FOR ${_name}: ${JSON.stringify(pc)}`);
    pcArray[_name] = pc;
    // callback(pc);
    callback(pcArray);
}
/**
 * Encender/apagar el micrófono
 */
export function toggleAudio() {
    if (localstream) {
        isEnableAudio = !isEnableAudio;
        localstream.getAudioTracks().forEach((track) => {
            track.enabled = isEnableAudio;
        });
    } else {
        console.log('in toggleAudio(), localstream is empty');
    }
    return isEnableAudio;
}
/**
 * Activar/desactivar vídeo
 */
export function toggleVideo() {
    if (localstream) {
        isEnableVideo = !isEnableVideo;
        localstream.getVideoTracks().forEach((track) => {
            track.enabled = isEnableVideo;
        });
    } else {
        console.log('in toggleVideo(), localstream is empty');
    }
    return isEnableVideo;
}
/**
 * cambiar de cámara
 *
 */
export function switchVideoType() {
    if (localstream) {
        localstream.getVideoTracks().forEach(track => {
            track._switchCamera();
        });
    } else {
        console.log('error');
    }
}
/**
 * Crear una transmisión de video local
 *
 * @param {*} isFront
 * @param {*} callback
 */

export function getStream(isFront, callback) {
    mediaDevices.enumerateDevices().then(sourceInfos => {
        //console.log('Log: '+ JSON.stringify(sourceInfos)); // -> Es un objecto de 3 
        let videoSourceId;
        for (let i = 0; i < sourceInfos.length; i++) {
            const sourceInfo = sourceInfos[i];
            if (sourceInfo.kind === 'videoinput' && sourceInfo.facing === (isFront ? 'front' : 'back')) {
                // if (sourceInfo.kind === 'videoinput' && sourceInfo.facing === (isFront ? 'front' : 'back')) { Was video only, not videoinput
                videoSourceId = sourceInfo.id;
            }
        }
        request(PERMISSIONS.IOS.CAMERA).then((response) => {
            if (response === 'granted') {
                request(PERMISSIONS.IOS.CAMERA).then((response) => {
                    if (response === 'granted') {
                        mediaDevices.getUserMedia({
                            audio: true,
                            video: {
                                width: 640,
                                height: 480,
                                frameRate: 30,
                                facingMode: (isFront ? "user" : "environment"),
                                deviceId: videoSourceId
                            }
                        })
                            .then(stream => {
                                callback(stream)
                            })
                            .catch(error => {
                                // Log error
                                console.log(error)
                            });
                    }
                })
            }
        })
    });
}
/**
 *
 * Crear una conexión WebRTC
 *
 * @param {*} sendMessage
 * @param {*} name
 * @param {*} isOffer
 * @param {*} options
 */
export function createPC(sendMessage, name, isOffer, options) {
    let pc = new RTCPeerConnection(ICE_CONFIG);
    pc.onnegotiationneeded = () => {
        console.log('onnegotiationneeded');
        if (isOffer) {
            isOffer = false;
            createOffer();
        }
    };
    pc.onicecandidate = (event) => {
        console.log('onicecandidate');
        if (event.candidate) {
            let msg = {
                event: 'candidate',
                // event: 'onIceCandidate',
                userid: socket.id,
                roomName: '9b33737f-737f-4a3d-a323-e1cd3f4b68b2',
                candidate: event.candidate
            };
            sendMessage(msg);
        }
    };
    pc.oniceconnectionstatechange = (event) => {
        // console.log('oniceconnectionstatechange');
        if (event.target.iceConnectionState === 'disconnected') {
            //localstream.release();
            // localstream = null;
            if (pc !== null) {
                pc.close();
                pc = null;
            }
        }
    };
    pc.onsignalingstatechange = (event) => {
        console.log('onsignalingstatechange: ', event.target.signalingState);
    };
    // send local stream
    pc.addStream(localstream);
    function createOffer() {
        // console.log('createOffer');
        pc.createOffer()
            .then(function (desc) {
                // console.log('...createOffer...');
                pc.setLocalDescription(desc);
                // something to do
                let msg = {
                    event: 'receiveVideoFrom',
                    userid: socket.id,
                    roomName: '9b33737f-737f-4a3d-a323-e1cd3f4b68b2',
                    // sdpOffer: desc.sdp,
                    sdpOffer: desc.sdp,
                };
                sendMessage(msg);
            })
            .catch(err => console.error(err));
    }
    return pc;
}
/**
 * Adición incremental de iceCandidate
 *
 * @param {*} name
 * @param {*} candidate
 */
export function addIceCandidate(name, candidate) {
    let pc = pcArray[name];
    if (pc) {
        pc.addIceCandidate(candidate);
    } else {
        console.log('pc.addIceCandidate failed : pc not exists');
    }
}
/**
 * Proceso SdpAnswer
 *
 * @param {*} name
 * @param {*} sdp
 * @param {*} callback
 */
export async function ProcessAnswer(name, sdp, callback) {
    let pc = pcArray[name];
    if (pc) {
        let answer = {
            type: 'answer',
            sdp: sdp,
        };
        if (pc) {
            // const remoteDesc = new RTCSessionDescription(answer);
            // await pc.setRemoteDescription(remoteDesc);

            await pc.setRemoteDescription(new RTCSessionDescription(answer)).then(function () {
                console.log('LOG:252 Answer = ' + answer);
                callback();
            })
                .catch(function (reason) {
                    callback(reason);
                    console.log('ERROR REASON:', reason);
                });
        }
    } else {
        console.log('ProcessAnswer failed : pc not exists');
    }
}
/**
 *
 * Cierra la conexión y libera el streamer local.
 *
 */


export function ViewPCArray (username) {
    // console.log(pcArray[username].getRemoteStreams()[0]);
    return pcArray
}


export function ReleaseMeidaSource() {
    if (localstream) {
        localstream.release();
        localstream = null;
    }
    if (pcArray !== null) {
        for (let mem in pcArray) {
            pcArray[mem].close();
            delete pcArray[mem];
        }
    }
}
function logError(error) {
    console.log('logError', error);
}

NodeJs Kurento

module.exports = function(io, socket) {
    // variables
const kurento = require('kurento-client');
const minimist = require('minimist');

var kurentoClient = null;
var iceCandidateQueues = {};


// constants
var argv = minimist(process.argv.slice(2), {
    default: {
        // as_uri: 'https://localhost:3000/',
        // ws_uri: 'ws://localhost:8888/kurento'
        as_uri: 'https://localhost:4000/',
        ws_uri: 'ws://localhost:8888/kurento'
    }
});

socket.on('message', function (message) {
    console.log('Message received: ', message.event);

    switch (message.event) {
        case 'joinRoom':
            joinRoom(socket, message.userName, message.roomName, err => {
                if (err) {
                    console.log(err);
                }
            });
            break;
        case 'receiveVideoFrom':
            receiveVideoFrom(socket, message.userid, message.roomName, message.sdpOffer, err => {
                if (err) {
                    console.log(err);
                }
            });
            break;
        case 'candidate':
            addIceCandidate(socket, message.userid, message.roomName, message.candidate, err => {
                if (err) {
                    console.log(err);
                }
            });
            break;
    }
    socket.on('disconnect', () => {
        leaveRoom(socket, message.roomName, err => {
            if (err) {
                console.log(err);
            };
        });
    })
});

// signaling functions
function joinRoom(socket, username, roomname, callback) {
    console.log(('Smeone in the Room').green + socket + " " + username + " " + roomname)
    getRoom(socket, roomname, (err, myRoom) => {
        if (err) {
            return callback(err);
        }

        myRoom.pipeline.create('WebRtcEndpoint', (err, outgoingMedia) => {
            outgoingMedia.setMaxVideoSendBandwidth(100);
            if (err) {
                return callback(err);
            }

            var user = {
                id: socket.id,
                name: username,
                outgoingMedia: outgoingMedia,
                incomingMedia: {}
            }

            let iceCandidateQueue = iceCandidateQueues[user.id];
            if (iceCandidateQueue) {
                while (iceCandidateQueue.length) {
                    let ice = iceCandidateQueue.shift();
                    console.error(`user: ${user.name} collect candidate for outgoing media`);
                    user.outgoingMedia.addIceCandidate(ice.candidate);
                }
            }

            user.outgoingMedia.on('OnIceCandidate', event => {
                let candidate = kurento.register.complexTypes.IceCandidate(event.candidate);
                socket.emit('message', {
                    event: 'candidate',
                    userid: user.id,
                    candidate: candidate
                });
            });

            socket.to(roomname).emit('message', {
                event: 'newParticipantArrived',
                userid: user.id,
                username: user.name
            });

            let existingUsers = [];
            for (let i in myRoom.participants) {
                if (myRoom.participants[i].id != user.id) {
                    existingUsers.push({
                        id: myRoom.participants[i].id,
                        name: myRoom.participants[i].name
                    });
                }
            }
            socket.emit('message', {
                event: 'existingParticipants',
                existingUsers: existingUsers,
                userid: user.id
            });
            myRoom.participants[user.id] = user;
        });
    });
}








// Disconnect from room
function leaveRoom(socket, roomname, callback) {

    if (io.sockets.adapter.rooms[roomname] == null) {
        // ROOMS
        // console.log((io.sockets.adapter.rooms))
        return;
    }

    var userSession = io.sockets.adapter.rooms[roomname].participants[socket.id];

    var myRoom = io.sockets.adapter.rooms[roomname] || { length: 0 };
    // MY ROOM
    // console.log(myRoom);


    if (!userSession) {
        return;
    }

    var room = io.sockets.adapter.rooms[roomname];

    if(!room){
        return;
    }

    console.log('notify all user that ' + userSession.name + ' is leaving the room ' + roomname);
    var usersInRoom = room.participants;
    delete usersInRoom[userSession.id];
    userSession.outgoingMedia.release();
    
    // release incoming media for the leaving user
    for (var i in userSession.incomingMedia) {
        userSession.incomingMedia[i].release();
        delete userSession.incomingMedia[i];
    }

    for (var i in usersInRoom) {
        var user = usersInRoom[i];
        // release viewer from this
        user.incomingMedia[userSession.id].release();
        delete user.incomingMedia[userSession.id];

        // notify all user in the room
        io.emit('message', {
            event: 'participantLeft',
            name: userSession.id
        });

        console.log('Mensaje emitido')
    }

    // Release pipeline and delete room when room is empty
    if (Object.keys(room.participants).length == 0) {
        room.pipeline.release();
        delete rooms[userSession.roomName];
    }
    delete userSession.roomName;
    // console.log(myRoom);
    callback(null);
}




function receiveVideoFrom(socket, userid, roomname, sdpOffer, callback) {
    getEndpointForUser(socket, roomname, userid, (err, endpoint) => {
        if (err) {
            return callback(err);
        }

        endpoint.processOffer(sdpOffer, (err, sdpAnswer) => {
            if (err) {
                return callback(err);
            }

            socket.emit('message', {
                event: 'receiveVideoAnswer',
                senderid: userid,
                sdpAnswer: sdpAnswer
            });

            endpoint.gatherCandidates(err => {
                if (err) {
                    return callback(err);
                }
            });
        });
    });
}

function addIceCandidate(socket, senderid, roomname, iceCandidate, callback) {
    // console.log(io.sockets.adapter.rooms);
    let user = io.sockets.adapter.rooms[roomname].participants[socket.id];
    if (user != null) {
        let candidate = kurento.register.complexTypes.IceCandidate(iceCandidate);
        if (senderid == user.id) {
            if (user.outgoingMedia) {
                user.outgoingMedia.addIceCandidate(candidate);
            } else {
                iceCandidateQueues[user.id].push({ candidate: candidate });
            }
        } else {
            if (user.incomingMedia[senderid]) {
                user.incomingMedia[senderid].addIceCandidate(candidate);
            } else {
                if (!iceCandidateQueues[senderid]) {
                    iceCandidateQueues[senderid] = [];
                }
                iceCandidateQueues[senderid].push({ candidate: candidate });
            }
        }
        callback(null);
    } else {
        callback(new Error("addIceCandidate failed"));
    }
}

// useful functions
function getRoom(socket, roomname, callback) {
    var myRoom = io.sockets.adapter.rooms[roomname] || { length: 0 };
    var numClients = myRoom.length;

    console.log(roomname, ' has ', numClients, ' clients');

    if (numClients == 0) {
        socket.join(roomname, () => {
            myRoom = io.sockets.adapter.rooms[roomname];
            getKurentoClient((error, kurento) => {
                kurento.create('MediaPipeline', (err, pipeline) => {
                    if (error) {
                        return callback(err);
                    }

                    myRoom.pipeline = pipeline;
                    myRoom.participants = {};
                    callback(null, myRoom);
                });
            });
        });
    } else {
        socket.join(roomname);
        callback(null, myRoom);
    }
}

function getEndpointForUser(socket, roomname, senderid, callback) {
    var myRoom = io.sockets.adapter.rooms[roomname];
    var asker = myRoom.participants[socket.id];
    var sender = myRoom.participants[senderid];

    if (asker.id === sender.id) {
        return callback(null, asker.outgoingMedia);
    }

    if (asker.incomingMedia[sender.id]) {
        sender.outgoingMedia.connect(asker.incomingMedia[sender.id], err => {
            if (err) {
                return callback(err);
            }
            callback(null, asker.incomingMedia[sender.id]);
        });
    } else {
        myRoom.pipeline.create('WebRtcEndpoint', (err, incoming) => {
            incoming.setMaxVideoSendBandwidth(100);
            if (err) {
                return callback(err);
            }

            asker.incomingMedia[sender.id] = incoming;

            let iceCandidateQueue = iceCandidateQueues[sender.id];
            if (iceCandidateQueue) {
                while (iceCandidateQueue.length) {
                    let ice = iceCandidateQueue.shift();
                    console.error(`user: ${sender.name} collect candidate for outgoing media`);
                    incoming.addIceCandidate(ice.candidate);
                }
            }

            incoming.on('OnIceCandidate', event => {
                let candidate = kurento.register.complexTypes.IceCandidate(event.candidate);
                console.log("CANDIDATE: ", event.candidate);
                socket.emit('message', {
                    event: 'candidate',
                    userid: sender.id,
                    candidate: candidate
                });
            });

            sender.outgoingMedia.connect(incoming, err => {
                if (err) {
                    return callback(err);
                }
                callback(null, incoming);
            });
        });
    }
}

function getKurentoClient(callback) {
    if (kurentoClient !== null) {
        return callback(null, kurentoClient);
    }

    kurento(argv.ws_uri, function (error, _kurentoClient) {
        if (error) {
            console.log("Could not find media server at address " + argv.ws_uri);
            return callback("Could not find media server at address" + argv.ws_uri
                + ". Exiting with error " + error);
        }

        kurentoClient = _kurentoClient;
        callback(null, kurentoClient);
    });
}


}
user8711448
  • 1
  • 1

0 Answers0