0

I am getting crazy. I am trying to save an array of packed record to disk to be read later on.

The following unit contain mainly two procedures.

  1. The first one InitSaveAndReLoad(), init a packed array of record, save it to the disk and reload from the disk in a new array of packed record and go through the loaded array and print the 20 first value. Works perfectly.
  2. The second one LoadFromFile(), just reload the array from disk. It is even call by InitSaveAndReload() and works perfectly as soon as the file has been created previously by the same instance of the application. I mean if I quit the application and relaunch, the LoadFromFile() procedure which just reload the file in an array of records does not works anymore. I don't understand why.

Any clue?

Thanks for your help. Already spend a full day on this issue and turning crazy!

unit Unit4;

interface

uses
  Winapi.Windows, Winapi.Messages, System.SysUtils, System.DateUtils,
  System.Variants, System.Classes, Vcl.Graphics,
  Vcl.Controls, Vcl.Forms, Vcl.Dialogs, Vcl.Menus, Vcl.StdCtrls;

type
  TRate = packed record
    time        : int64;
    open        : double;
    low         : double;
    high        : double;
    close       : double;
    tick_volume : int64;
    spread      : integer;
    real_volume : int64;
  end;

  PRate = ^TRate;

  TForm4 = class(TForm)
    MemoLogs: TMemo;
    SaveDialog1: TSaveDialog;
    edFile: TEdit;
    Button1: TButton;
    Button2: TButton;
    Button3: TButton;
    Button4: TButton;
    procedure InitSaveAndReload(Sender: TObject);
    procedure Reload(Sender: TObject);
    procedure SelectFile(Sender: TObject);
    procedure LoadFromFile();
    procedure Button4Click(Sender: TObject);
  private
    { Private declarations }
  public
    { Public declarations }
  end;

var
  Form4: TForm4;

implementation

{$R *.dfm}

function TimeElapsedToString(time : int64; show_ms : boolean = false) : string;
var
  TmpVal:real;
  TmpStr:string;
begin
  TmpVal := time;
  TmpStr := '';

  TmpVal := TmpVal / 3600000;
  TmpStr := inttostr(trunc(TmpVal));
  if Length(TmpStr) = 1 then TmpStr := '0' + TmpStr;


  TmpVal := (TmpVal-trunc(TmpVal))* 3600000;
  TmpVal := TmpVal / 60000;
  if TmpVal<10 then
     TmpStr := TmpStr + ':0' + inttostr(trunc(TmpVal))
  else
     TmpStr := TmpStr + ':' + inttostr(trunc(TmpVal));

  TmpVal := (TmpVal-trunc(TmpVal))*60000;
  TmpVal := TmpVal / 1000;
  if TmpVal<10 then
     TmpStr := TmpStr + ':0' + inttostr(trunc(TmpVal))
  else
     TmpStr := TmpStr + ':' + inttostr(trunc(TmpVal));

  if show_ms then
  begin
    TmpVal := (TmpVal-trunc(TmpVal))*1000;
    TmpVal := TmpVal;
    if TmpVal<10 then
      TmpStr := TmpStr + ':00' + inttostr(trunc(TmpVal))
    else if TmpVal<100 then
      TmpStr := TmpStr + ':0' + inttostr(trunc(TmpVal))
    else
      TmpStr := TmpStr + ':' + inttostr(trunc(TmpVal));

  end;

  Result := TmpStr;
end;

procedure TForm4.SelectFile(Sender: TObject);
begin
  if SaveDialog1.Execute then
     edFile.Text := SaveDialog1.FileName;
end;

procedure TForm4.Button4Click(Sender: TObject);
begin
  MemoLogs.Lines.Clear;
end;

procedure TForm4.InitSaveAndReload(Sender: TObject);
var
  _start   : TDatetime;
  ARate    : PRate;
  filename : string;
  Stream   : TFileStream;
  i,L      : integer;
  rates_M1 : array of PRate;
  //rates    : array of PRate;
begin
  filename := edFile.Text;

  MemoLogs.Lines.Add('Initialization of 7 million array of records... Please wait.');
  Refresh;

  // init array
  _start := Now;
  SetLength(rates_M1, 7000000);
  for i:= 0 to 6999999 do
  begin
    New(ARate);
    ARate.time        := DateTimeToUnix(IncMinute(Now, i));
    ARate.open        := 1.25698;
    ARate.low         := 1.2574;
    ARate.high        := 1.2547;
    ARate.close       := 1.65874;
    ARate.tick_volume := 154;
    ARate.spread      := 5;
    ARate.real_volume := 15741;
    rates_M1[i]       := ARate;
  end;
  MemoLogs.Lines.Add(IntToStr(Length(rates_M1)) + ' array of records initialized ' + TimeElapsedToString(MilliSecondsBetween(Now, _start), true));

  // save array
  _start:= Now;
  Stream:= TFileStream.Create(filename , fmCreate);
  try
    L:= Length(rates_M1);
    Stream.WriteBuffer(L, SizeOf(L));
    Stream.WriteBuffer(Pointer(rates_M1)^, L * SizeOf(ARate));
  finally
    Stream.Free;
    MemoLogs.Lines.Add(IntToStr(Length(rates_M1)) + ' records saved to disk in ' + TimeElapsedToString(MilliSecondsBetween(Now, _start), true));
  end;

  LoadFromFile();

end;

procedure TForm4.LoadFromFile;
var
  _start   : TDatetime;
  ARate    : PRate;
  filename : string;
  Stream   : TFileStream;
  i,L      : integer;
  rates    : array of PRate;
begin
  // reload array
  _start := Now;
  filename := edFile.Text;
  Stream:= TFileStream.Create(filename , fmOpenRead);
  try
    Stream.Read(L, SizeOf(L));
    //SetLength(rates_M1, L);
    // even use another empty array of ARate to be sure I am not using the same filled array!
    SetLength(rates, L);
    // I don't want to parse all records...
//    for i := 0 to L-1 do
//      begin
//        Stream.Read(rates_M1[i].AID, SizeOf(ARecord.AID));
//        Stream.Read(rates_M1[i].time, SizeOf(ARecord.time));
//      end;
    Stream.Read(Pointer(rates)^, L * SizeOf(ARate));
  finally
    Stream.Free;
    MemoLogs.Lines.Add(IntToStr(Length(rates)) + ' records loaded from disk in ' + TimeElapsedToString(MilliSecondsBetween(Now, _start), true));
  end;

  // Print 20 first records just reloaded!
  MemoLogs.Lines.Add('Print 20 first records just reloaded in another array of records!' + TimeElapsedToString(MilliSecondsBetween(Now, _start), true));
  for i := 0 to 20 do
     MemoLogs.Lines.Add('i=' + IntToStr(i) + #9
                      + IntToStr(rates[i].time) + #9
                      + FloatToStr(rates[i].open) + #9
                       );

end;

procedure TForm4.Reload(Sender: TObject);
begin
  LoadFromFile();
end;

end.

Result

When I say 'Does not works anymore', I mean once you called InitSaveAndReload() procedure, you can call LoadFromFile() as many time as you want, but if you to call this procedure just after launching the app, trying to use an old file created by the InitSaveAndReload procedure, then it does not work the same!

The Unit provide is as simple as possible. Just create a new projet add 3 buttons a TMemo and one TEdit. If I could join a .rar I would have enclosed the project...

och
  • 33
  • 3
  • You said: *...does not work anymore...*. In what way does it not work? We can't see your screen or possible erroneous behaviour, you know! – Tom Brunberg Jan 09 '18 at 10:44
  • Delete all the irrelevant code in your question and create a ***minimal*** [mcve]. – Disillusioned Jan 09 '18 at 10:48
  • 1
    You are writing the addresses of the records rather than their contents. There are a great many other problems, but that's the first one. Packing your record is a big mistake. That just leads to it being misaligned. – David Heffernan Jan 09 '18 at 10:59
  • 2
    Next time you post a question, please be sure to not include the words *it doesn't work* unless you describe what you expect it to do and what it does indtead. Did you look at the file e.g. in a hex editor, does it look as you expect? An MVCE in this case doesn't need to have a anything more than the record type def, an functions to create a few records, save the records and read the records. – Tom Brunberg Jan 09 '18 at 11:16
  • @David : Thanks for your reply. I understood I should loop through the array and save each member, to save values instead of adress, but in that case it is much more longer (5 minutes for 7M records). What would you suggest to improve performance of saving such kind of structure? – och Jan 09 '18 at 11:21
  • Performance is hardly the issue. You are saving the address of the record rather than saving the record. Why did you do that? I for one don't understand why you individually `New` each record. I can't give you performance advice with no understanding of the motivation behind this. You should only really consider performance when you are more proficient. – David Heffernan Jan 09 '18 at 11:27
  • Also, with some elementary debugging you would have found the problem faster than it took you post the question, yet less than the "already spend a full day". – Tom Brunberg Jan 09 '18 at 11:27

1 Answers1

2

You are saving the address of each record rather than saving contents of the record. You have an array of pointers, and you save that array to file. With your current data structure you would need to save each record individually, because the actual data does not lie contiguously in memory.

David Heffernan
  • 601,492
  • 42
  • 1,072
  • 1,490
  • Thanks for your reply. I change my code and now I loop through the array and save each member of the record. I am now able to save and reload correctly the array of record using the InitSaveAndReload and LoadFromFile procedure with no issu, but it is quit time consuming 5 minutes to save and 4:30 minutes to reload. Any suggestion to improve those performance? – och Jan 09 '18 at 11:55
  • Use a [buffered file stream](https://stackoverflow.com/questions/5639531/buffered-files-for-faster-disk-access/5639712#5639712), or store the data contiguously in an array of `TRate`. – David Heffernan Jan 09 '18 at 12:01
  • Thanks, David for your insigth, I will try asap and keep you updated... – och Jan 09 '18 at 12:19
  • There no updating needing. Hasn't the question you asked been answered? – David Heffernan Jan 09 '18 at 12:28
  • I guess so, Actually I am trying to replace my array of pointer by an array of TRate, then I will try implementing buffered file stream. Than I will close the question, but you definitly put me on the rigth way! – och Jan 09 '18 at 12:31
  • Once you have an array of record it is stored continuously. Then you can write it one giant blog and the performance will good. You won't need explicit buffering then. I'm sure this question can be marked as answered – David Heffernan Jan 09 '18 at 13:22
  • How would you 'write it one giant blog'? – och Jan 09 '18 at 13:33
  • should have been "block" not "blog". Which is what the code in the question does. But once you change to an array of record rather than array of pointer to record, then that code would be fine. Anyway, we have moved away from the question you asked, and I'd like to stop at this point. I really do think I answered the question that you asked. Did I not? – David Heffernan Jan 09 '18 at 13:33
  • Thanks a lot! Have a nice day. – och Jan 09 '18 at 13:36
  • 1
    New users should be forced to at least take the tour, so they would have some insight in how SO works. Maybe by allowing questions only after the tour – Tom Brunberg Jan 09 '18 at 14:01
  • I don't think any site tour will help anyone making a correlation with what he/she has asked and what he/she expects to be answered. That's just personality. – Sertac Akyuz Jan 09 '18 at 20:05