Getting HTML from response stream using Indy's IDTCPClient

Question

I have a component that a previous employee has created. It uses Indy (IDTCPClient) and the following method to perform requests (Where "aReadHeader" is a predefined Uri built before being passed in).

function TMyConnector.GET(aRawHeader: String): String;
begin
if Not Connected then Connected := True;
if Connected then
  begin
  FRawRequest :=  'GET /'+ aRawHeader + ' HTTP/'+HTTPVerText+#13#10+
                  'Host: '+FHost+#13#10+

                  'Cookie: UserHPos=IOGLO00003000090000C000BS; '+
                  'LOSID=qsBiy/wEDCq6tOXFzGbOlTD1lmo5AXdFnCkbzzPn6+qCeheYVyTcumRrjsqh+Hds4Fr2gZDazfDzGN1RA+nnHuQQeBy78ZUgctrZyyy9MnGl2qI/ulkV6EPxAfmmLg/lopRq99f5gAcG/dgtytAJjS+aD5DqtHGrAqjiqgtkwuA=; '+
                  'LoginHPos=IOGLO00003000090000C000BS; '+
                  'UIHPos=IOGLO00003000020000500003; '+
                  'LOSG=61939308-7C83-47ED-B909-2D2D10AD7026; '+
                  'fControllingBusiness=IOGLO000030000900001000050000200001'+#13#10+

                  'Connection: Close'+#13#10+
                  #13#10;

  FSock.Socket.Write(FRawRequest);
  FRawResponse := FSock.Socket.ReadLn(#13#10#13#10,nil);
  Result := FRawResponse;
  if ResponseStream = nil then ResponseStream := TMemoryStream.Create
    else ResponseStream.SetSize(0);
  FSock.Socket.ReadStream(ResponseStream,-1,True);
  if Connected and (Not KeepAlive) then Connected := False;
  end;
end;

Question the FRawResponse returns

HTTP/1.0 200 OK Content-Length: 5560 Date: Mon, 18 Nov 2013 15:05:07 GMT Content-Type: text/html; charset=UTF-8 ...,public

How can I actually get this html content from ResponseStream to HTML

One of the methods which currently exists is "GenerateJSON" (see code below). I would like to create one called "GenerateHTML"

Function StreamToArray(Strm:TStream):TArray<Byte>;
Begin
 Strm.Position := 0;
 SetLength(Result,Strm.Size);
 Strm.Read(Result[0],Strm.Size);
End;


Procedure TMyConnector.GenerateJSON;
begin
 if ResponseStream <> nil then
  Begin
   ResponseJSON_V := TJSONObject.ParseJSONValue(StreamToArray(ResponseStream),0) as TJSONValue; // Note ResponseJSON_V is declared as TJSONValue in TMyConnector);
  End;
end;

so, I would need

Procedure TMyConnector.GenerateHTML;
begin
 if ResponseStream <> nil then
  Begin
   // result:= html from stream here
  End;
end;

EDIT:

Procedure TMyConnector.GenerateXML;
var
 S: String;
begin
if ResponseStream <> nil then
  Begin
   try
    while FSock.IOHandler.CheckForDataOnSource(30) do
    begin
       S := FSock.IOHandler.InputBufferAsString;
    end;
  finally
    ResponseStr_v:= S;
  end;
  End;
end;

Answer 1

The previous employee obviously did not know about the existence of Indy's TIdHTTP component, or how Indy's ReadStream() method actually works, or how HTTP actually works in general. You should re-write the function to use TIdHTTP and let it handle all of the details for you, eg:

function TMyConnector.GET(aRawHeader: String): String;
begin
  FHTTP.ProtocolVersion := ...; // pv1_0 or pv1_1

  // Indy has a TIdCookieManager component that can be attached to
  // the TIdHTTP.CookieManager property, which handles parsing,
  // tracking, and sending back cookies for you for each HTTP
  // request, so this should be re-written to utilize that
  // functionality...
  //
  FHTTP.Request.CustomHeaders.Values['Cookie'] := 'UserHPos=IOGLO00003000090000C000BS; '+
  'LOSID=qsBiy/wEDCq6tOXFzGbOlTD1lmo5AXdFnCkbzzPn6+qCeheYVyTcumRrjsqh+Hds4Fr2gZDazfDzGN1RA+nnHuQQeBy78ZUgctrZyyy9MnGl2qI/ulkV6EPxAfmmLg/lopRq99f5gAcG/dgtytAJjS+aD5DqtHGrAqjiqgtkwuA=; '+
  'LoginHPos=IOGLO00003000090000C000BS; '+
  'UIHPos=IOGLO00003000020000500003; '+
  'LOSG=61939308-7C83-47ED-B909-2D2D10AD7026; '+
  'fControllingBusiness=IOGLO000030000900001000050000200001';

  FHTTP.Request.Connection := 'Close';

  if ResponseStream = nil then
    ResponseStream := TMemoryStream.Create
  else
    ResponseStream.Size := 0;

  try
    try
      FHTTP.Get('http://' + FHost + '/' + aRawHeader, ResponseStream);
    finally
      FRawResponse := FHTTP.Response.RawHeaders.Text;
      Result := FRawResponse;
    end;
  except
    on E: EIdHTTPProtocolException do
    begin
      // HTTP error response. You can grab the error content if you need it...
      WriteStringToStream(ResponseStream, E.ErrorMessage);
    end;
    on E: Exception do
    begin
      // some other error, handle as needed...
    end;
  end;
end;

If a re-write to use TIdHTTP is not possible, then you must update the original code to actually implement rudimentary HTTP parsing. It is not enough to simply call ReadStream(), you have to know WHEN to call it, and WHAT parameters to pass to it. Try something more like this:

function TMyConnector.GET(aRawHeader: String): String;
var
  Headers: TIdHeaderList;
  Size: integer;

  function InternalReadLn: String;
  begin
    Result := FSock.IOHandler.ReadLn;
    if FSock.IOHandler.ReadLnTimedout then begin
      raise EIdReadTimeout.Create('');
    end;
  end;

  function ChunkSize: integer;
  var
    j: Integer;
    s: string;
  begin
    s := InternalReadLn;
    j := Pos(';', s); {do not localize}
    if j > 0 then begin
      s := Copy(s, 1, j - 1);
    end;
    Result := StrToInt('$' + Trim(s), 0);
  end;

begin
  if Not Connected then Connected := True;
  if Connected then
  begin
    FRawRequest :=  'GET /'+ aRawHeader + ' HTTP/'+HTTPVerText+#13#10+
                    'Host: '+FHost+#13#10+

                    'Cookie: UserHPos=IOGLO00003000090000C000BS; '+
                    'LOSID=qsBiy/wEDCq6tOXFzGbOlTD1lmo5AXdFnCkbzzPn6+qCeheYVyTcumRrjsqh+Hds4Fr2gZDazfDzGN1RA+nnHuQQeBy78ZUgctrZyyy9MnGl2qI/ulkV6EPxAfmmLg/lopRq99f5gAcG/dgtytAJjS+aD5DqtHGrAqjiqgtkwuA=; '+
                    'LoginHPos=IOGLO00003000090000C000BS; '+
                    'UIHPos=IOGLO00003000020000500003; '+
                    'LOSG=61939308-7C83-47ED-B909-2D2D10AD7026; '+
                    'fControllingBusiness=IOGLO000030000900001000050000200001'+#13#10+

                    'Connection: Close'+#13#10+
                    #13#10;

    FSock.IOHandler.Write(FRawRequest);
    FRawResponse := FSock.IOHandler.ReadLn(#13#10#13#10,nil);
    Result := FRawResponse;

    if ResponseStream = nil then ResponseStream := TMemoryStream.Create
    else ResponseStream.Size := 0;

    Headers := TIdHeaderList.Create(QuoteHTTP);
    try
      Headers.Text := FRawResponse;

      if Pos('chunked', LowerCase(Headers.Values['Transfer-Encoding']) > 0 then
      begin
        Size := ChunkSize;
        while Size <> 0 do begin
          FSock.IOHandler.ReadStream(ResponseStream, Size);
          InternalReadLn;
          Size := ChunkSize;
        end;
        repeat until InternalReadLn = '';
      end
      else if Headers.IndexOfName('Content-Length') <> -1 then
      begin
        FSock.IOHandler.ReadStream(ResponseStream, StrToInt64(Headers.Values['Content-Length']), False);
      end
      else
        FSock.IOHandler.ReadStream(ResponseStream, -1, True);
    finally
      Headers.Free;
    end;

    if Connected and (Not KeepAlive) then Connected := False;
  end;
end;