0

I wrote some code to serialize an object containing some data, save it, and then load it next time the program is run. (Using a BinaryFormater)

However the program takes an insanely long time to actually load the data (at 100 entrys, it takes around 15-30 seconds), and I need the file to store much more at around 300,000 entrys. The save time isn't great either but its much faster then loading. I'd like to know what sort of options there are for saving and loading database's and the differences between them. I have read that Xml is pretty slow, I also would like to know how lazy Loading works (load only as needed), and what file formats it can be applied to.

Here is the code that saves and then repopulates the data:

static void SaveAsBinary(string fullpath, object data)
        {
            // Create the new, empty data file.
            string fileName = fullpath;
            if (File.Exists(fileName))
            {
                Console.WriteLine(fileName + @" already exists!");
                if (File.Exists(fileName + ".bak"))
                { File.Delete(fileName + ".bak"); }
                File.Move(fileName, fileName + ".bak");
            }
            try
            {
                BinaryFormatter bf = new BinaryFormatter();
                FileStream fs = new FileStream(fileName, FileMode.CreateNew);
                bf.Serialize(fs, data);
                fs.Flush(true);
                fs.Dispose();
            }
            catch (Exception ex)
            {
                Console.WriteLine("Exception: " + ex.Message);
            }
        }
        private void PopulateDatabase(SaveDatabaseIndex db)
        {
            if (last_page < cur_page) db.current_page = -1;
            db.UrlList = UrlList;
            db.firstentryOnSave = firstentryOnload;
            foreach (string url in UrlList)
            {
                eh_entry ent = getEntry(url);
                details_PaneSaveData detpane = new details_PaneSaveData
                {
                    rating = ent.detailspanel.rating,
                    uploader = ent.detailspanel.uploader,
                    category = ent.detailspanel.category,
                    date = ent.detailspanel.date,
                    filecount = ent.detailspanel.filecount,
                    filesize = ent.detailspanel.filesize,
                    tags = ent.detailspanel.tags
                };
                eh_entrySave entSaveData = new eh_entrySave
                {
                    dl_location = ent.dl_location,
                    files = ent.file,
                    title = ent.title,
                    detail_SaveData = detpane
                };
                db.eh_Save.Add(url, entSaveData);
            }
            SaveAsBinary(Application.StartupPath + "\\" + db.db_name + ".dat", db);
        }
        public void LoadDataBase(string DatabasePath)
        {
            SaveDatabaseIndex LoadedData = LoadAsBinary(DatabasePath) as SaveDatabaseIndex;
            //Populate Required Objects with Loaded Data
            if (LoadedData != null)
            {
                cur_page = LoadedData.current_page;
                firstentryOnload = LoadedData.firstentryOnSave;
                UrlList = LoadedData.UrlList;
                Parallel.ForEach(UrlList, entUrl =>
                {
                    eh_entrySave ent_Save = LoadedData.eh_Save[entUrl] as eh_entrySave;
                    HTEntry.Add(entUrl, new eh_entry
                    {
                        detailspanel = new details_Pane
                        {
                            rating = ent_Save.detail_SaveData.rating,
                            uploader = ent_Save.detail_SaveData.uploader,
                            category = ent_Save.detail_SaveData.category,
                            date = ent_Save.detail_SaveData.date,
                            filecount = ent_Save.detail_SaveData.filecount,
                            filesize = ent_Save.detail_SaveData.filesize,
                            tags = ent_Save.detail_SaveData.tags
                        },
                        dl_location = ent_Save.dl_location,
                        title = ent_Save.title
                    });
                });

            }
        }
        static object LoadAsBinary(string fullpath)
        {
            string fileName = fullpath;
            if (File.Exists(fileName))
            {
                try
                {
                    FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read);
                    BinaryFormatter bf = new BinaryFormatter();
                    var loadedData = bf.Deserialize(fs);
                    return loadedData;
                }
                catch (Exception ex)
                {
                    Console.WriteLine("Exception: " + ex.Message);
                }
            }
            else if (File.Exists(fileName + "bak"))
                fileName = fileName + "bak";
                try
                {
                    FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read);
                    BinaryFormatter bf = new BinaryFormatter();
                    var loadedData = bf.Deserialize(fs);
                    return loadedData;
                }
                catch (Exception ex)
                {
                    Console.WriteLine("Exception: " + ex.Message);
                }
            }
            return null;
        }

Edit: Just to clarify I am looking for an already existing fileformat or DB Product that would be good to use instead of using Binary Serialization.

Edit2: Well my question was not worded great and I ended up finding an answer to my problem here Which database would you recommend to use with C# (.NET) application?

Community
  • 1
  • 1
Jason Brown
  • 127
  • 2
  • 13
  • 1
    Not very clear what advice you are looking for. It looks like you trying to write some sort of database - while entertaining it is not exactly practical as there are couple existing well established DB products out there. You may want to search for "profiling C# applications" using your favorite search engine to find issues with your code. – Alexei Levenkov Oct 31 '15 at 01:48
  • @AlexeiLevenkov I am looking for an already existing fileformat or DB Product that would be good to use instead of using Binary Serialization. I have no idea what i should be searching, since when I search FileFormat and C# it generally just gives information about the fileformats Used by C#. – Jason Brown Oct 31 '15 at 01:55
  • Search for tools/libraries/products/tutorial is off-topic on SO. "C# database" may be better search term... – Alexei Levenkov Oct 31 '15 at 02:07
  • @AlexeiLevenkov Ah, ok. Well Telling me to use profiling was quite helpful so thankyou. Although I am still unsure of where to go from here, Binary serialization is taking to long. And I don't know of other solutions. Is there a place on StackExchange that allows for search questions (mainly libraries). Since I still don't know what the most effective Database type would be, I have heard about SQL so I will probably look into that... Anyway Thank you – Jason Brown Oct 31 '15 at 02:14
  • There is http://softwarerecs.stackexchange.com/ , but you need very concrete requirements to ask question there. "A database I can use with C#" is not going to be received well. – Alexei Levenkov Oct 31 '15 at 02:22
  • @AlexeiLevenkov I found exactly what I needed =) I'm not sure this question was very good, But I really wasn't sure how to word it. You gave me quite a bit of good advice. Searched "A database I can use with C#" and got some good ideas from it. So Thank you Very much! – Jason Brown Oct 31 '15 at 02:30

0 Answers0