I managed to create a TCP client app that streams video to a server app using Unity and Sockets with the great help of Programmer from this answer: https://stackoverflow.com/a/42727918/8713780
Now I need, while streaming the video, to send a simple text message from the server to the client, which triggers the client to send a response message back to the server right away.
The first try is almost always a success but on the second try, the response message to the server causing the video to get stuck, the message itself is corrupted and I get this exception: "System.OverflowException: Number overflow. at <0x00000> "
This is the server script:
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using System.Net.Sockets;
using System.Net;
using System.Text;
using System;
using UnityEngine.UI;
using System.IO;
public class MySocketsServer : MonoBehaviour {
TcpListener listner;
TcpClient client;
NetworkStream stream;
StreamWriter streamWriter;
StreamReader streamReader;
int port = 8010;
int SEND_RECEIVE_COUNT = 15;
bool getVideo = false;
bool waitingForClient = false;
bool serverConnected = false;
bool clientConnected = false;
private List<TcpClient> clients = new List<TcpClient>();
WaitForEndOfFrame waitEndOfFrame = new WaitForEndOfFrame();
WaitForFixedUpdate waitFixedUpdate = new WaitForFixedUpdate ();
Texture2D tex;
public RawImage serverRawImage;
public GameObject startServerUiBtn;
public GameObject stopServerUiBtn;
public GameObject startStopTrackingUiBtn;
public Text startStopTrackingTxt;
void Start(){
Application.runInBackground = true;
tex = new Texture2D(2, 2, TextureFormat.RGB24, false);
}
public void StartServer () {
listner = new TcpListener(IPAddress.Any, port);
client = new TcpClient ();
listner.Start();
serverConnected = true;
Debug.Log ("Server conected! - IP = " + Network.player.ipAddress);
UIDebugLog.VisableLog ("Server conected! - IP = " + Network.player.ipAddress);
StartCoroutine (WaitForClient());
startServerUiBtn.SetActive (false);
}
public void StopServer(){
stopServerUiBtn.SetActive (false);
startStopTrackingUiBtn.SetActive (false);
serverConnected = false;
getVideo = false;
streamWriter.Close ();
stream.Close ();
client.Close ();
foreach (TcpClient c in clients) {
c.Close ();
}
if (listner != null)
{
listner.Stop();
}
startServerUiBtn.SetActive (true);
serverRawImage.gameObject.SetActive (false);
}
private void OnApplicationQuit()
{
StopServer ();
}
IEnumerator WaitForClient()
{
clientConnected = false;
waitingForClient = true;
stopServerUiBtn.SetActive (true);
// Wait for client to connect in another Thread
Loom.RunAsync(() =>
{
while (waitingForClient)
{
// Wait for client connection
client = listner.AcceptTcpClient();
// We are connected
clients.Add(client);
clientConnected = true;
}
});
//Wait until client has connected
while (!clientConnected)
{
yield return null;
}
stream = client.GetStream ();
streamWriter = new StreamWriter(stream);
streamReader = new StreamReader(stream);
streamWriter.AutoFlush = true;
startStopTrackingUiBtn.SetActive (true);
serverRawImage.gameObject.SetActive (true);
Debug.Log("Client is Connected!");
UIDebugLog.VisableLog("Client is Connected!");
waitingForClient = false;
getVideo = true;
imageReceiver ();
}
int imageSize;
void imageReceiver()
{
Loom.RunAsync(() =>
{
while (getVideo)
{
//Read Image Count
imageSize = readImageByteSize(SEND_RECEIVE_COUNT);
//Read Image Bytes and Display it
readFrameByteArray(imageSize);
}
});
}
byte[] imageBytesCount;
private int readImageByteSize(int size)
{
bool disconnected = false;
try{
imageBytesCount = new byte[size];
var total = 0;
do
{
var read = stream.Read(imageBytesCount, total, size - total);
if (read == 0)
{
disconnected = true;
break;
}
total += read;
} while (total != size);
}catch(Exception e){
Debug.Log ("readImageByteSize() error: " + e);
disconnected = true;
}
if (disconnected)
{
return -1;
}
else
{
return frameByteArrayToByteLength(imageBytesCount);
}
}
//Converts the byte array to the data size and returns the result
int frameByteArrayToByteLength(byte[] frameBytesLength)
{
return BitConverter.ToInt32(frameBytesLength, 0);
}
byte[] imageBytes;
private void readFrameByteArray(int size)
{
try{
bool disconnected = false;
imageBytes = new byte[size];
var total = 0;
do
{
var read = stream.Read(imageBytes, total, size - total);
if (read == 0)
{
disconnected = true;
break;
}
total += read;
} while (total != size);
bool readyToReadAgain = false;
if (!disconnected)
{
//Display Image on the main Thread
Loom.QueueOnMainThread(() =>
{
displayReceivedImage(imageBytes);
readyToReadAgain = true;
});
}
//Wait until old Image is displayed
while (!readyToReadAgain)
{
System.Threading.Thread.Sleep(1);
}
}catch(Exception e){
Debug.Log ("readFrameByteArray() error: " + e);
}
}
void displayReceivedImage(byte[] receivedImageBytes)
{
tex.LoadImage(receivedImageBytes);
serverRawImage.texture = tex;
}
bool waitingForMsgCallback = false;
public void SendMsgToTheClient(){
try{
// Stop recieving video for a moment:
getVideo = false;
// Start listenung to the callback:
waitingForMsgCallback = true;
StartCoroutine (RecieveMsgCallback());
// send the msg:
streamWriter.WriteLine ("msg_from_the_server");
Debug.Log("msg_from_the_server sended");
UIDebugLog.VisableLog("msg_from_the_server sended");
}catch(Exception e){
Debug.Log("SendMsgToTheClient() error: " + e);
}
}
IEnumerator RecieveMsgCallback(){
while (waitingForMsgCallback) {
try{
if (stream.DataAvailable) {
// get the callback msg:
string callbackMsg = streamReader.ReadLine ();
Debug.Log ("Recieved msg Callback: " + callbackMsg);
UIDebugLog.VisableLog ("Recieved msg Callback: " + callbackMsg);
waitingForMsgCallback = false;
// Continue receiving video:
getVideo = true;
imageReceiver ();
}
}catch(Exception e){
Debug.Log("RecieveStartTrackingCallback() error: " + e);
}
yield return waitFixedUpdate;
}
}
}
And the Client script:
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using System.Net.Sockets;
using System.Text;
using System.Net;
using UnityEngine.UI;
using System;
using System.IO;
public class MySocketsClient : MonoBehaviour {
TcpClient client;
NetworkStream stream;
StreamWriter streamWriter;
StreamReader streamReader;
int port = 8010;
int SEND_RECEIVE_COUNT = 15;
string IP = "127.0.0.1";
bool connected = false;
bool sendingVideo = false;
bool waitingForMsgFromServer = false;
WebCamTexture webCam;
public RawImage clientRawImage;
Texture2D currentTexture;
WaitForEndOfFrame waitEndOfFrame = new WaitForEndOfFrame();
WaitForFixedUpdate waitFixedUpdate = new WaitForFixedUpdate();
public GameObject StartClientUiBtn;
public GameObject StopClientUIBtn;
public InputField ipField;
Rect captureRect;
void Start () {
Application.runInBackground = true;
if (PlayerPrefs.GetString ("server_address") != "") {
IP = PlayerPrefs.GetString ("server_address");
ipField.text = IP;
}
captureRect = new Rect (0, 0, Screen.width, Screen.height);
InitWebCam();
currentTexture = new Texture2D(Screen.width, Screen.height, TextureFormat.RGB24, false);
frameBytesLength = new byte[SEND_RECEIVE_COUNT];
}
public void StartClientBtn(){
if (IP == "") {
Debug.Log ("Please enter an ip address");
return;
}
StartCoroutine (StartClient());
StartClientUiBtn.SetActive (false);
}
IEnumerator StartClient(){
client = new TcpClient();
StopClientUIBtn.SetActive (true);
//Connect to server from another Thread
Loom.RunAsync(() =>
{
Debug.Log("Connecting to server...");
IPAddress ipaddress = IPAddress.Parse(IP);
client.Connect(ipaddress, port);
Debug.Log("Connected!");
connected = true;
});
while (!connected) {
yield return null;
}
stream = client.GetStream();
streamWriter = new StreamWriter(stream);
streamReader = new StreamReader(stream);
streamWriter.AutoFlush = true;
//Start sending video:
sendingVideo = true;
StartCoroutine(senderCOR());
//Start listening to msg from the server:
waitingForMsgFromServer = true;
StartCoroutine(RecieveServerMsg());
}
public void StopClient(){
StopClientUIBtn.SetActive (false);
connected = false;
sendingVideo = false;
streamReader.Close ();
streamWriter.Close ();
stream.Close ();
if (client != null)
{
client.Close();
}
if (webCam != null && webCam.isPlaying)
{
webCam.Stop();
}
StartClientUiBtn.SetActive (true);
}
void OnApplicationQuit()
{
StopClient ();
}
public void OnChangeAddress(){
IP = ipField.text;
PlayerPrefs.SetString ("server_address", IP);
}
void InitWebCam(){
webCam = new WebCamTexture();
webCam.requestedHeight = 10;
webCam.requestedWidth = 10;
clientRawImage.texture = webCam;
webCam.Play();
}
bool readyToGetFrame = false;
byte[] frameBytesLength;
byte[] jpgBytes;
IEnumerator senderCOR()
{
readyToGetFrame = true;
while (sendingVideo)
{
//Wait for End of frame
yield return waitEndOfFrame;
currentTexture.ReadPixels (captureRect , 0, 0);
currentTexture.Apply();
jpgBytes = currentTexture.EncodeToJPG(12);
//Fill total byte length to send. Result is stored in frameBytesLength
byteLengthToFrameByteArray(jpgBytes.Length, frameBytesLength);
readyToGetFrame = false;
try{
Loom.RunAsync(() =>
{
//NetworkStream videoStream = client.GetStream();
//Send total byte count first
stream.Write(frameBytesLength, 0, frameBytesLength.Length);
//Send the image bytes
stream.Write(jpgBytes, 0, jpgBytes.Length);
//Sent. Set readyToGetFrame true
readyToGetFrame = true;
});
}catch(Exception e){
Debug.Log ("senderCOR() error: " + e.Message);
readyToGetFrame = true;
}
//Wait until we are ready to get new frame(Until we are done sending data)
while (!readyToGetFrame)
{
yield return null;
}
}
}
//Converts the data size to byte array and put result to the fullBytes array
void byteLengthToFrameByteArray(int byteLength, byte[] fullBytes)
{
//Clear old data
Array.Clear(fullBytes, 0, fullBytes.Length);
//Convert int to bytes
byte[] bytesToSendCount = BitConverter.GetBytes(byteLength);
//Copy result to fullBytes
bytesToSendCount.CopyTo(fullBytes, 0);
}
IEnumerator RecieveServerMsg(){
while (waitingForMsgFromServer) {
try{
if (stream.DataAvailable) {
// read the msg from the server:
string msg = streamReader.ReadLine ();
UIDebugLog.VisableLog ("ReadStartTrackingMsg: " + msg);
// send a msg callback to the server:
streamWriter.WriteLine ("Callback Message from the client");
UIDebugLog.VisableLog ("callback sended...");
}
}catch(Exception e){
Debug.Log ("ReadStartTrackingMsg() error: " + e.Message);
}
yield return waitFixedUpdate;
yield return waitEndOfFrame;
}
}
}
The final client app should run on Android and the server app should run on PC. I've tested on both and got the same results.
Any advice will be most appreciated
Thanks!
