My dataset: 500,000 points in 960 dimensions. The size of the file is 1.9 GB (1,922,000,000 bytes).
The code works for smaller data sets, but for this it will crash in the same point every time. Here is a minimal example:
#include <iostream>
#include <vector>
template<typename T>
class Division_Euclidean_space {
public:
/**
* The data type.
*/
typedef T FT;
/**
* Constructor, which
* sets 'N' and 'D' to zero.
*/
Division_Euclidean_space()
: N(0),
D(0) {
}
/**
* @param n - size of data
*/
void setSize(size_t& n) {
N = n;
}
/**
* @param n - size of data
*/
void setSize(int n) {
N = n;
}
/**
* Get the number of points
*
* @return - the number of points
*/
const size_t& size() const {
return N;
}
/**
* Get the dimension of points
*
* @return - the dimension
*/
const int dim() {
return D;
}
/**
* @param d - dimension of data
*/
void setDim(int& d) {
D = d;
}
/**
* \brief Inserts a new value to the collection of
* points, held in the private vector.
*
* @param v - value to be inserted
*/
void insert(FT v) {
p.push_back(v);
}
private:
/**
* number of points
*/
size_t N;
/**
* dimension of points
*/
int D;
/**
* vector of points
* Note that indexing is of the form: [i * D + j]
*/
std::vector<FT> p;
};
typedef Division_Euclidean_space<int> Division_space;
typedef Division_space::FT FT;
template<typename T>
void readDivisionSpacefvecs(Division_Euclidean_space<T>& ds, int& N, int& D,
char* filename) {
FILE* fid;
fid = fopen(filename, "rb");
if (!fid)
printf("I/O error : Unable to open the file %s\n", filename);
// we assign the return value of fread() to 'sz' just to suppress a warning
size_t sz = fread(&D, sizeof(D), 1, fid);
fseek(fid, 0L, SEEK_END);
sz = ftell(fid);
N = sz / (1 * 4 + D * 4);
//printf("N = %d, D = %d, |%s|\n", N, D, filename);
fseek(fid, 0L, SEEK_SET);
ds.setSize(N);
ds.setDim(D);
std::cout << ds.dim() << " " << ds.size() << "\n";
int c = 0;
float v;
int i, j;
for (i = 0; i < N; ++i) {
sz = fread(&D, sizeof(D), 1, fid);
//printf("%d\n", D);
for (j = 0; j < D; ++j) {
sz = fread(&v, sizeof(v), 1, fid);
if (c >= 279619)
printf("j = %d, v = %f, read up to point %d\n", j, v, c);
ds.insert(v);
}
++c;
printf("read up to %d\n", c);
}
if (c != N)
printf("WARNING! Read less points than expected.\n");
}
int main() {
Division_space test;
int N, D;
readDivisionSpacefvecs<FT>(test, N, D, "../../parallel/rkd_forest/Datasets/gist/gist_learn.fvecs");
return 0;
}
Output:
...
j = 255, v = 0.052300, read up to point 279620
j = 256, v = 0.052300, read up to point 279620
terminate called after throwing an instance of 'std::bad_alloc'
what(): std::bad_alloc
Aborted
Am I out of memory? How can I know?
Here is how much memory I have:
samaras@samaras-A15:~$ free -mt
total used free shared buffers cached
Mem: 3934 2638 1295 0 179 1000
-/+ buffers/cache: 1458 2475
Swap: 3987 0 3987
Total: 7922 2638 5283
