2

I'm reading a file in chucks of registers of arrays in a size of 4096.

I have a container array outside of the loop and i want to 'append' the chunks of data to it in sequential order, What is the fastest and most efficient way to do so?

PointRecord4 * AllRegisters = new PointRecord4[ptr->Number_of_Point_records];

    if (ptr->Point_Data_Format_ID == 4)
        switch ((*ptr).Point_Data_Format_ID)
        {
        case 4:
            {
                PointRecord4 registers[4096];
                ULONG32 pointsToRead = 4096;
                ULONG32 pointsLoaded = 0;               
                ULONG32 i;

                for (i = 0; pointsLoaded < ptr->Number_of_Point_records; i += pointsToRead)
                {
                    pointsToRead =  std::min(pointsToRead, ptr->Number_of_Point_records - pointsLoaded);
                    input_file.read(PTR(&registers), sizeof(PointRecord4) * pointsToRead);
                    pointsLoaded += pointsToRead;

                    //Insert registers[4096] into AllRegisters, Appending it
                } 
            } break;
    }

I've tried to simply iterate over 4096 and just push back to the vector but this was incredible slow.

below is my full code:

#include "vgl.h"
#include <fstream>
#include <iostream>
#include <vector>
#include <string.h>


using namespace std;

#define PTR reinterpret_cast<char *>

#pragma pack(1)
typedef struct LasHeader
{
    char           File_Signature[4];                            // 4 Bytes
    UINT16 FileSourceID;                                         // 2 Bytes
    UINT16 Global_Encoding;                                      // 2 Bytes
    ULONG32  GUID1;                                              // 4 Bytes
    UINT16 GUID2;                                                // 2 Bytes
    UINT16 GUID3;                                                // 2 Bytes
    UCHAR  GUID4[8];                                             // 2 Bytes
    UCHAR  Version_Major;                                        // 1 Bytes
    UCHAR  Version_Minor;                                        // 1 Bytes
    CHAR           System_Identifier[32];                        // 32 Bytes
    CHAR           Generating_Software[32];                      // 32 Bytes
    UINT16 File_Creation_Day_Of_Year;                            // 2 Bytes
    UINT16 File_Creation_Year;                                   // 2 Bytes
    UINT16 Header_Size;                                          // 2 Bytes
    ULONG32 Offset_to_point_data;                                // 4 Bytes
    ULONG32 Number_of_Variable_Length_Records;                   // 4 Bytes
    CHAR Point_Data_Format_ID;                                   // 1 Bytes
    UINT16 Point_Data_Record_Length;                             // 2 Bytes
    ULONG32 Number_of_Point_records;                             // 4 Bytes 
    ULONG32 Number_of_points_by_return[5];                       // 20 Bytes / Verified

    double X_Scale_Factor;                                       // 8 bytes  / Verified
    double Y_Scale_Factor;                                       // 8 bytes
    double Z_Scale_Factor;                                       // 8 bytes
    double X_Offset;                                             // 8 bytes
    double Y_Offset;                                             // 8 bytes
    double Z_Offset;                                             // 8 bytes
    double Max_X;                                                // 8 bytes
    double Min_X;                                                // 8 bytes
    double Max_Y;                                                // 8 bytes
    double Min_Y;                                                // 8 bytes
    double Max_Z;                                                // 8 bytes
    double Min_Z;                                                // 8 bytes

    #ifdef FULLWAVE
        unsigned long long Start_of_Waveform_Data_Packet_Record; // 8 bytes
    #endif

    union
    {
        char* MemoryBlock;
    };

} *header;

struct PointRecord4
{
    UINT32 X;
    UINT32 Y;
    UINT32 Z;
    UINT16 Intensity;
    CHAR BitMask;
    CHAR Classification;
    CHAR ScanAngleRank;
    CHAR UserData;
    UINT16 PointSourceID;
    double GPSTime;
    CHAR WPDI;
    UINT64 WFOffset;
    UINT32 WFPacketSize;
    float WFReturnLocation;
    float WFXt;
    float WFYt;
    float WFZt;
};

extern header ptr = new LasHeader;

HRESULT OpenLasFile(char *path)
{
    HRESULT result;
    ifstream input_file(path, ios::in | ios::binary);
    result = GetLastError();

    input_file.read(PTR(ptr), sizeof(LasHeader));
    result = GetLastError();
    input_file.seekg (ptr->Offset_to_point_data, input_file.beg);

    PointRecord4 * AllRegisters = new PointRecord4[ptr->Number_of_Point_records];

    if (ptr->Point_Data_Format_ID == 4)
        switch ((*ptr).Point_Data_Format_ID)
        {
        case 4:
            {
                PointRecord4 registers[4096];
                ULONG32 pointsToRead = 4096;
                ULONG32 pointsLoaded = 0;               
                ULONG32 i;

                for (i = 0; pointsLoaded < ptr->Number_of_Point_records; i += pointsToRead)
                {
                    pointsToRead =  std::min(pointsToRead, ptr->Number_of_Point_records - pointsLoaded);
                    input_file.read(PTR(&registers), sizeof(PointRecord4) * pointsToRead);
                    pointsLoaded += pointsToRead;

                    //Insert registers[4096] into AllRegisters, Appending it
                } 
            } break;
    }

    input_file.close();

    return result;
}
Dean
  • 499
  • 6
  • 13
  • 34

2 Answers2

0

For you particular case, you can save the copy by just read into the target buffer:

input_file.read(AllRegisters + pointsLoaded, sizeof(PointRecord4) * pointsToRead);

In other cases where copy is necessary, memcpy is your best bet to copy large chunk of POD data.

X.J
  • 662
  • 3
  • 6
0

It's unclear why you're not just reading directly into AllRegisters.

However, I suspect that the append-stage is not the bottleneck. When you are reading data in large chunks like this, it pays to use unbuffered I/O. Disk I/O will be your limiting factor here, not memory I/O.

See my post regarding buffered vs unbuffered I/O in C.

Here is a post dealing with unbuffered I/O on std::ifstream

Community
  • 1
  • 1
paddy
  • 60,864
  • 6
  • 61
  • 103