DataSnap ServerMethod functions returned as ftStream parameters being wrongly truncated

Question

As DataSnap users will know, its ServerMethods return values to their callers as DataSnap parameters.

There have been a number of reports on SO and elsewhere relating to a problem with DataSnap servers returning ServerMethod results as ftStream parameters, that the stream is truncated prematurely or returned empty. An example is here: Can't retrieve TStreams bigger than around 260.000 bytes from a Datasnap Server

I have put together a reproducible test case of this that I intend submitting to Emba's Quality Portal as an MCVE, but before I do I'd like some help pinning down where the problem occurs. I'm using Delphi Seattle on Win64, compiling to 32-bits, btw.

My MCVE is completely self-contained (i.e. includes both server and client) and does not depend on any database data. Its ServerMethods module contains a function (BuildString in the code below) which returns a string of a caller-specified length and two ServerMethods GetAsString and GetAsStream which return the result as parameters of types ftString and ftStream, respectively.

Its GetString method successfully returns a string of any requested length up to the maximum I've tested, which is 32000000 (32 million) bytes.

Otoh, the GetStream method works up to a requested size of 30716; above that, the returned stream has a size of -1 and is empty. The expected behaviour of course that it should be capable of working with much larger sizes, just as GetString does.

On the outbound (server) side, at some point the returned stream is passed into DataSnap's JSon layer en route to the tcp/ip transport layer and on the inbound side, similarly, the stream is retrieved from the JSon layer. What I'd like to be able to do, and what this q is about, is to capture the outbound and inbound JSon representations of the AsStream parameter value in human-legible form so that I identify whether the unwanted truncation of its data occurs on the server or client side. How do I do that?

the reason I'm asking this is that despite hours of looking I've been unable to identify exactly where the JSon conversions occur. It's like looking for a needle in a haystack. If you take a look at the method TDBXJSonStreamWriter.WriteParameter in Data.DBXStream, the one thing it doesn't write is the stream's contents!

One thing I have been able to establish is regarding line 4809 in Data.DBXStream

Size := ((FBuf[IncrAfter(FOff)] and 255) shl 8) or (FBuf[IncrAfter(FOff)] and 255)

in the function TDBXRowBuffer.ReadReaderBlobSize. On entry to this method, Size is initialised to zero, and it is this line which sets Size to 30716 for all requested stream sizes >= that value. But I don't know whether this is cause or effect, i.e. whether the stream trucation has already taken place or whether it's this line which causes it.

My code is below; apologies for the length of it, but DataSnap projects require quite a lot of baggage at the best of times and I've included some code which initialises some of the components to avoid having to post .DFMs too.

ServerMethods code:

unit ServerMethods2u;
interface
uses System.SysUtils, System.Classes, System.Json, variants, Windows,
    Datasnap.DSServer, Datasnap.DSAuth, DataSnap.DSProviderDataModuleAdapter;

{$MethodInfo on}
type
  TServerMethods1 = class(TDSServerModule)
  public
    function GetStream(Len: Integer): TStream;
    function GetString(Len: Integer): String;
  end;
{$MethodInfo off}

implementation

{$R *.dfm}

uses System.StrUtils;

function BuildString(Len : Integer) : String;
var
  S : String;
  Count,
  LeftToWrite : Integer;
const
  scBlock = '%8d bytes'#13#10;
begin
  LeftToWrite := Len;
  Count := 1;
  while Count <= Len do begin
    S := Format(scBlock, [Count]);
    if LeftToWrite >= Length(S) then
    else
      S := Copy(S, 1, LeftToWrite);
    Result := Result + S;
    Inc(Count, Length(S));
    Dec(LeftToWrite, Length(S));
  end;
  if Length(Result) > 0 then
    Result[Length(Result)] := '.'
end;

function TServerMethods1.GetStream(Len : Integer): TStream;
var
  SS : TStringStream;
begin
  SS := TStringStream.Create;
  SS.WriteString(BuildString(Len));
  SS.Position := 0;
  Result := SS;
end;

function TServerMethods1.GetString(Len : Integer): String;
begin
  Result := BuildString(Len);
end;

ServerContainer code:

unit ServerContainer2u;
interface
uses System.SysUtils, System.Classes, Datasnap.DSTCPServerTransport,
  Datasnap.DSServer, Datasnap.DSCommonServer, Datasnap.DSAuth, IPPeerServer,
  DataSnap.DSProviderDataModuleAdapter;

type
  TServerContainer1 = class(TDataModule)
    DSServer1: TDSServer;
    DSTCPServerTransport1: TDSTCPServerTransport;
    DSServerClass1: TDSServerClass;
    procedure DataModuleCreate(Sender: TObject);
    procedure DSServerClass1GetClass(DSServerClass: TDSServerClass;
      var PersistentClass: TPersistentClass);
  end;

var
  ServerContainer1: TServerContainer1;

implementation

{$R *.dfm}

uses ServerMethods2u;

procedure TServerContainer1.DataModuleCreate(Sender: TObject);
begin
  DSServerClass1.Server := DSServer1;
  DSTCPServerTransport1.Server := DSServer1;
end;

procedure TServerContainer1.DSServerClass1GetClass(
  DSServerClass: TDSServerClass; var PersistentClass: TPersistentClass);
begin
  PersistentClass := TServerMethods1;
end;

end.

ServerForm code:

unit ServerForm2u;
interface
uses Winapi.Windows, Winapi.Messages, System.SysUtils, System.Variants,
  System.Classes, Vcl.Graphics, Vcl.Controls, Vcl.Forms, Vcl.Dialogs,
  Vcl.StdCtrls, DBXJSON, Data.DBXDataSnap, IPPeerClient,
  Data.DBXCommon, Data.FMTBcd, Data.DB, Data.SqlExpr, Data.DbxHTTPLayer,
  DataSnap.DSServer;

type
  TForm1 = class(TForm)
    btnGetStream: TButton;
    edStreamSize: TEdit;
    SQLConnection1: TSQLConnection;
    SMGetStream: TSqlServerMethod;
    Memo1: TMemo;
    Label1: TLabel;
    btnGetString: TButton;
    Label2: TLabel;
    edStringSize: TEdit;
    SMGetString: TSqlServerMethod;
    procedure FormCreate(Sender: TObject);
    procedure btnGetStreamClick(Sender: TObject);
    procedure btnGetStringClick(Sender: TObject);
  private
  public
  end;

var
  Form1: TForm1;

implementation

{$R *.dfm}

procedure TForm1.FormCreate(Sender: TObject);
begin

  SqlConnection1.ConnectionData.Properties.Values['CommunicationProtocol'] := 'tcp/ip';
  SqlConnection1.ConnectionData.Properties.Values['BufferKBSize'] := '64';

  SMGetStream.Params.Clear;
  SMGetStream.Params.CreateParam(ftInteger, 'Len', ptInput);
  SMGetStream.Params.CreateParam(ftStream, 'Result', ptOutput);

  SMGetString.Params.Clear;
  SMGetString.Params.CreateParam(ftInteger, 'Len', ptInput);
  SMGetString.Params.CreateParam(ftString, 'Result', ptOutput);

end;

procedure TForm1.btnGetStreamClick(Sender: TObject);
var
  SS : TStringStream;
  S : TStream;
begin
  Memo1.Lines.Clear;
  SS := TStringStream.Create;
  try
    SMGetStream.Params[0].AsInteger := StrtoInt(edStreamSize.Text);
    SMGetStream.ExecuteMethod;
    S := SMGetStream.Params[1].AsStream;
    S.Position := 0;
    if S.Size > 0 then begin
      try
        SS.CopyFrom(S, S.Size);
        Memo1.Lines.BeginUpdate;
        Memo1.Lines.Text := SS.DataString;
        Memo1.Lines.Insert(0, IntToStr(S.Size));
      finally
        Memo1.Lines.EndUpdate;
      end;
    end
    else
      ShowMessage(IntToStr(S.Size));
  finally
    SS.Free;
  end;
end;

procedure TForm1.btnGetStringClick(Sender: TObject);
var
  S : String;
  Size : Integer;
begin
  Memo1.Lines.Clear;
  Size :=  StrtoInt(edStringSize.Text);
  SMGetString.Params[0].AsInteger := Size;
  SMGetString.ExecuteMethod;
  S := SMGetString.Params[1].AsString;
  if Length(S) > 0 then begin
    try
      Memo1.Lines.BeginUpdate;
      Memo1.Lines.Text := S;
      Memo1.Lines.Insert(0, IntToStr(Length(S)));
    finally
      Memo1.Lines.EndUpdate;
    end;
  end;
end;

end.