I wrote some code to serialize an object containing some data, save it, and then load it next time the program is run. (Using a BinaryFormater)
However the program takes an insanely long time to actually load the data (at 100 entrys, it takes around 15-30 seconds), and I need the file to store much more at around 300,000 entrys. The save time isn't great either but its much faster then loading. I'd like to know what sort of options there are for saving and loading database's and the differences between them. I have read that Xml is pretty slow, I also would like to know how lazy Loading works (load only as needed), and what file formats it can be applied to.
Here is the code that saves and then repopulates the data:
static void SaveAsBinary(string fullpath, object data)
{
// Create the new, empty data file.
string fileName = fullpath;
if (File.Exists(fileName))
{
Console.WriteLine(fileName + @" already exists!");
if (File.Exists(fileName + ".bak"))
{ File.Delete(fileName + ".bak"); }
File.Move(fileName, fileName + ".bak");
}
try
{
BinaryFormatter bf = new BinaryFormatter();
FileStream fs = new FileStream(fileName, FileMode.CreateNew);
bf.Serialize(fs, data);
fs.Flush(true);
fs.Dispose();
}
catch (Exception ex)
{
Console.WriteLine("Exception: " + ex.Message);
}
}
private void PopulateDatabase(SaveDatabaseIndex db)
{
if (last_page < cur_page) db.current_page = -1;
db.UrlList = UrlList;
db.firstentryOnSave = firstentryOnload;
foreach (string url in UrlList)
{
eh_entry ent = getEntry(url);
details_PaneSaveData detpane = new details_PaneSaveData
{
rating = ent.detailspanel.rating,
uploader = ent.detailspanel.uploader,
category = ent.detailspanel.category,
date = ent.detailspanel.date,
filecount = ent.detailspanel.filecount,
filesize = ent.detailspanel.filesize,
tags = ent.detailspanel.tags
};
eh_entrySave entSaveData = new eh_entrySave
{
dl_location = ent.dl_location,
files = ent.file,
title = ent.title,
detail_SaveData = detpane
};
db.eh_Save.Add(url, entSaveData);
}
SaveAsBinary(Application.StartupPath + "\\" + db.db_name + ".dat", db);
}
public void LoadDataBase(string DatabasePath)
{
SaveDatabaseIndex LoadedData = LoadAsBinary(DatabasePath) as SaveDatabaseIndex;
//Populate Required Objects with Loaded Data
if (LoadedData != null)
{
cur_page = LoadedData.current_page;
firstentryOnload = LoadedData.firstentryOnSave;
UrlList = LoadedData.UrlList;
Parallel.ForEach(UrlList, entUrl =>
{
eh_entrySave ent_Save = LoadedData.eh_Save[entUrl] as eh_entrySave;
HTEntry.Add(entUrl, new eh_entry
{
detailspanel = new details_Pane
{
rating = ent_Save.detail_SaveData.rating,
uploader = ent_Save.detail_SaveData.uploader,
category = ent_Save.detail_SaveData.category,
date = ent_Save.detail_SaveData.date,
filecount = ent_Save.detail_SaveData.filecount,
filesize = ent_Save.detail_SaveData.filesize,
tags = ent_Save.detail_SaveData.tags
},
dl_location = ent_Save.dl_location,
title = ent_Save.title
});
});
}
}
static object LoadAsBinary(string fullpath)
{
string fileName = fullpath;
if (File.Exists(fileName))
{
try
{
FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read);
BinaryFormatter bf = new BinaryFormatter();
var loadedData = bf.Deserialize(fs);
return loadedData;
}
catch (Exception ex)
{
Console.WriteLine("Exception: " + ex.Message);
}
}
else if (File.Exists(fileName + "bak"))
fileName = fileName + "bak";
try
{
FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read);
BinaryFormatter bf = new BinaryFormatter();
var loadedData = bf.Deserialize(fs);
return loadedData;
}
catch (Exception ex)
{
Console.WriteLine("Exception: " + ex.Message);
}
}
return null;
}
Edit: Just to clarify I am looking for an already existing fileformat or DB Product that would be good to use instead of using Binary Serialization.
Edit2: Well my question was not worded great and I ended up finding an answer to my problem here Which database would you recommend to use with C# (.NET) application?