Sending a Dynamic array (Inside a record) through Socket?

Question

i'm trying to transfer a record from server to client, directly using .SendBuf().

however, this record has a member which is a dynamic array, and i have read somewhere (here in SOF) that when sending records, the members must be STATIC (fixed-length), but the problem is... i cannot determine how many arguments i would send (in the future).

how can i solve this problem ?

procedure TServerClass.SendBufToSocket(const vmName: TVMNames; const vmArgs: Array of TValue);
var
  // this record is sent to client
  // vmName = method to be called [in]
  // vmArgs = Argument for the method [in, optional]
  BufRec: packed record
    vmName: array[0..49] of char;
    vmArgs: Array of TValue;
  end;

  s: string;
  i: integer;
begin
  // convert enum method name to string
  s:= GetEnumName(TypeInfo(TVMNames), Integer(vmName));

  // copy method name to record
  lstrcpy(BufRec.vmName, pChar(s));

  // copy arg array to record
  SetLength(BufRec.vmArgs, length(vmArgs));

  for i:=0 to high(vmArgs)
    do BufRec.vmArgs[i] := vmArgs[i];

  // send record
  ServerSocket.Socket.Connections[idxSocket].SendBuf(PByte(@BufRec)^, SizeOf(BufRec));
end;

I found out from where i've read it, here: ReceiveBuf from TCustomWinSocket won't work with dynamic arrays for the buffer

Answer 1

You will not be able to send the record as-is, so in fact you don't even need to use a record at all. You must serialize your data into a flat format that is suitable for transmission over a network. For example, when sending a string, send the string length before sending the string data. Likewise, when sending an array, send the array length before sending the array items. As for the items themselves, since TValue is dynamic, you have to serialize it into a flat format as well.

Try something like this on the sending side:

procedure TServerClass.SendBufToSocket(const vmName: TVMNames; const vmArgs: Array of TValue);
var
  I: integer;

  procedure SendRaw(Data: Pointer; DataLen: Integer);
  var
    DataPtr: PByte;
    Socket: TCustomWinSocket;
    Sent, Err: Integer;
  begin
    DataPtr := PByte(Data);
    Socket := ServerSocket.Socket.Connections[idxSocket];
    while DataLen > 0 do
    begin
      Sent := Socket.SendBuf(DataPtr^, DataLen);
      if Sent > 0 then
      begin
        Inc(DataPtr, Sent);
        Dec(DataLen, Sent)
      end else
      begin
        Err := WSAGetLastError();
        if Err <> WSAEWOULDBLOCK then
          raise Exception.CreateFmt('Unable to sent data. Error: %d', [Err]);
        Sleep(10);
      end;
    end;
  end;

  procedure SendInteger(Value: Integer);
  begin
    Value := htonl(Value);
    SendRaw(@Value, SizeOf(Value));
  end;

  procedure SendString(const Value: String);
  var
    S: UTF8string;
    Len: Integer;
  begin
    S := Value;
    Len := Length(S);
    SendInteger(Len);
    SendRaw(PAnsiChar(S), Len);
  end;

begin
  SendString(GetEnumName(TypeInfo(TVMNames), Integer(vmName)));
  SendInteger(Length(vmArgs));
  for I := Low(vmArgs) to High(vmArgs) do
    SendString(vmArgs[I].ToString);
end;

And then on the receiving side:

type
  TValueArray := array of TValue;

procedure TServerClass.ReadBufFromSocket(var vmName: TVMNames; var vmArgs: TValueArray);
var
  Cnt, I: integer;
  Tmp: String;

  procedure ReadRaw(Data: Pointer; DataLen: Integer);
  var
    DataPtr: PByte;
    Socket: TCustomWinSocket;
    Read, Err: Integer;
  begin
    DataPtr := PByte(Data);
    Socket := ClientSocket.Socket;
    while DataLen > 0 do
    begin
      Read := Socket.ReceiveBuf(DataPtr^, DataLen);
      if Read > 0 then
      begin
        Inc(DataPtr, Read);
        Dec(DataLen, Read);
      end
      else if Read = 0 then
      begin
        raise Exception.Create('Disconnected');
      end else
      begin
        Err := WSAGetLastError();
        if Err <> WSAEWOULDBLOCK then
          raise Exception.CreateFmt('Unable to read data. Error: %d', [Err]);
        Sleep(10);
      end;
    end;
  end;

  function ReadInteger: Integer;
  begin
    ReadRaw(@Result, SizeOf(Result));
    Result := ntohl(Result);
  end;

  function ReadString: String;
  var
    S: UTF8String;
    Len: Integer;
  begin
    Len := ReadInteger;
    SetLength(S, Len);
    ReadRaw(PAnsiChar(S), Len);
    Result := S;
  end;

begin
  vmName := TVMNames(GetEnumValue(TypeInfo(TVMNames), ReadString));
  Cnt := ReadInteger;
  SetLength(vmArgs, Cnt);
  for I := 0 to Cnt-1 do
  begin
    Tmp := ReadString;
    // convert to TValue as needed...
    vmArgs[I] := ...;
  end;
end;

With that said, note that socket programming is more complex than this simple example shows. You have to do proper error handling. You have to account for partial data sends and receives. And if you are using non-blocking sockets, if the socket enters a blocking state then you have to wait for it to enter a readable/writable state again before you can attempt to read/write data that is still pending. You are not doing any of that yet. You need to get yourself a good book on effective socket programming.

Update: if you are trying to utilize the OnRead and OnWrite events of the socket components, you have to take a different approach:

procedure TServerClass.ClientConnect(Sender: TObject; Socket: TCustomWinSocket);
begin
  Socket.Data := TMemoryStream.Create;
end;

procedure TServerClass.ClientDisconnect(Sender: TObject; Socket: TCustomWinSocket);
begin
  TMemoryStream(Socket.Data).Free;
  Socket.Data := nil;
end;

procedure TServerClass.ClientWrite(Sender: TObject; Socket: TCustomWinSocket);
var
  OutBuffer: TMemoryStream;
  Ptr: PByte;
  Sent, Len: Integer;
begin
  OutBufer := TMemoryStream(Socket.Data);
  if OutBuffer.Size = 0 then Exit;

  OutBuffer.Position := 0;
  Ptr := PByte(OutBuffer.Memory);

  Len := OutBuffer.Size - OutBuffer.Position;
  while Len > 0 do
  begin
    Sent := Socket.SendBuf(Ptr^, Len);
    if Sent <= 0 then Break;
    Inc(Ptr, Sent);
    Dec(Len, Sent)
  end;

  if OutBuffer.Position > 0 then
  begin
    if OutBuffer.Position >= OutBuffer.Size then
      OutBuffer.Clear
    else
    begin
      Move(Ptr^, OutBuffer.Memory^, Len);
      OutBuffer.Size := Len;
    end;
  end;
end;

procedure TServerClass.SendBufToSocket(const vmName: TVMNames; const vmArgs: Array of TValue);
var
  I: integer;
  Socket: TCustomWinSocket;
  OutBuffer: TMemoryStream;

  procedure SendRaw(Data: Pointer; DataLen: Integer);
  var
    DataPtr: PByte;
    Sent: Integer;
  begin
    if DataLen < 1 then Exit;
    DataPtr := PByte(Data);
    if OutBuffer.Size = 0 then
    begin
      repeat
        Sent := Socket.SendBuf(DataPtr^, DataLen);
        if Sent < 1 then Break;
        Inc(DataPtr, Sent);
        Dec(DataLen, Sent)
      until DataLen < 1;
    end;
    if DataLen > 0 then
    begin
      OutBuffer.Seek(0, soEnd);
      OutBuffer.WriteBuffer(DataPtr^, DataLen);
    end;
  end;

  procedure SendInteger(Value: Integer);
  begin
    Value := htonl(Value);
    SendRaw(@Value, SizeOf(Value));
  end;

  procedure SendString(const Value: String);
  var
    S: UTF8string;
    Len: Integer;
  begin
    S := Value;
    Len := Length(S);
    SendInteger(Len);
    SendRaw(PAnsiChar(S), Len);
  end;

begin
  Socket := ServerSocket.Socket.Connections[idxSocket];
  OutBuffer := TMemoryStream(Socket.Data);

  SendString(GetEnumName(TypeInfo(TVMNames), Integer(vmName)));
  SendInteger(Length(vmArgs));
  for I := Low(vmArgs) to High(vmArgs) do
    SendString(vmArgs[I].ToString);
end;

And then on the receiving side:

procedure TServerClass.ClientConnect(Sender: TObject; Socket: TCustomWinSocket);
begin
  Socket.Data := TMemoryStream.Create;
end;

procedure TServerClass.ClientDisconnect(Sender: TObject; Socket: TCustomWinSocket);
begin
  TMemoryStream(Socket.Data).Free;
  Socket.Data := nil;
end;

procedure TServerClass.ClientRead(Sender: TObject; Socket: TCustomWinSocket);
var
  InBuffer: TMemoryStream;      
  Ptr: PByte;
  OldSize, Pos, Read: Integer;

  function HasAvailable(DataLen: Integer): Boolean;
  being
    Result := (InBuffer.Size - InBuffer.Position) >= DataLen;
  end;

  function ReadInteger(var Value: Integer);
  begin
    Result := False;
    if HasAvailable(SizeOf(Integer)) then
    begin
      InBuffer.ReadBuffer(Value, SizeOf(Integer));
      Value := ntohl(Value);
      Result := True;
    end;
  end;

  function ReadString(var Value: String);
  var
    S: UTF8String;
    Len: Integer;
  begin
    Result := False;
    if not ReadInteger(Len) then Exit;
    if not HasAvailable(Len) then Exit;
    SetLength(S, Len);
    InBuffer.ReadBuffer(PAnsiChar(S)^, Len);
    Value := S;
    Result := True;
  end;

  function ReadNames: Boolean;
  var
    S: String;
    vmName: TVMNames;
    vmArgs: TValueArray;
  begin
    Result := False;
    if not ReadString(S) then Exit;
    vmName := TVMNames(GetEnumValue(TypeInfo(TVMNames), S));
    if not ReadInteger(Cnt) then Exit;
    SetLength(vmArgs, Cnt);
    for I := 0 to Cnt-1 do
    begin
      if not ReadString(S) then Exit;
      // convert to TValue as needed...
      vmArgs[I] := ...;
    end;
    // use vmArgs as needed...
    Result := True;
  end;

begin
  InBuffer := TMemoryStream(Socket.Data);

  Read := Socket.ReceiveLength;
  if Read <= 0 then Exit;

  OldSize := InBuffer.Size;
  InBuffer.Size := OldSize + Read;

  try
    Ptr := PByte(InBuffer.Memory);
    Inc(Ptr, OldSize);
    Read := Socket.ReceiveBuf(Ptr^, Read);
  except
    Read := -1;
  end;

  if Read < 0 then Read := 0;
  InBuffer.Size := OldSize + Read;
  if Read = 0 then Exit;

  InBuffer.Position := 0;

  repeat
    Pos := InBuffer.Position;
  until not ReadNames;

  InBuffer.Position := Pos;
  Read := InBuffer.Size - InBuffer.Position;
  if Read < 1 then
    InBuffer.Clear
  else
  begin
    Ptr := PByte(InBuffer.Memory);
    Inc(Ptr, InBuffer.Position);
    Move(Ptr^, InBuffer.Memory^, Read);
    InBuffer.Size := Read;
  end;
end;

Answer 2

As mentioned in some comments, serialize your record to a stream and then send the stream contents over the wire. I use kbLib in some of my projects and it works really good. You can use any dynamic type like strings, arrays in your record.

Small example:

type 
  TMyRecord = record
    str : string;
  end;

procedure Test;

var
 FStream : TMemoryStream;
 MYrecord : TMyRecord;
 MYrecord1 : TMyRecord;

begin
 FStream := TMemoryStream.Create;
 try
  MyRecord.Str := 'hello world';
  // save record to stream 
  TKBDynamic.WriteTo(FStream, MyRecord, TypeInfo(TMyRecord)); 
  FStream.Position := 0;
  // read record from stream
  TKBDynamic.ReadFrom(FStream, MyRecord1, TypeInfo(TMyRecord));  
  If MyRecord1.Str <> MyRecord.Str then
   ShowMessage('this should not happen!');
  finally
   FStream.Free; 
  end;
end;