why my boost serialization failed with input stream error?

Question

I have a complex class, which need to serialization.

I use boost.serialization.

The class is a singleton class, with several function. singleton is not the vital part, because i have tried some simple code with simple singleton class, it worked well.

// static_player.h
#pragma once
#include <map>
#include <unordered_set>
#include <vector>
#include <memory>
#include <boost/archive/text_oarchive.hpp>
#include <boost/archive/text_iarchive.hpp>
#include <boost/serialization/vector.hpp>
#include <boost/serialization/unordered_map.hpp>
#include <boost/serialization/unordered_set.hpp>
#include <boost/serialization/map.hpp>

#include "util/file_util.h"
#include "struct/struct_util.hpp"

namespace util:trade_util {
    typedef std::unordered_map<std::string, std::unordered_map<std::string, std::unordered_map<std::string, double>>> NTKD; // map of name_ticker_key_value
    typedef std::unordered_map<std::string, std::unordered_map<std::string, std::unordered_map<std::string, std::string>>> NTKS; // map of name_ticker_key_value
    typedef std::unordered_map<std::string, std::map<int32_t, std::unordered_map<std::string, std::unordered_map<std::string, double>> > > NDTKD;
    typedef std::unordered_map<std::string, std::map<int32_t, std::unordered_map<std::string, std::unordered_map<std::string, std::string>> > > NDTKS;
    
    class StaticPlayer {
     private:
      StaticPlayer();
      StaticPlayer(const StaticPlayer&) = delete;
      StaticPlayer(const StaticPlayer&&) = delete;
      virtual ~StaticPlayer();
 
      class DM {
       public:
        DM() {}
        DM(NTKD * numeric_data, NTKS * string_data) : numeric_data_(numeric_data), string_data_(string_data) {}
        virtual ~DM() = default;
        inline double get_numeric(const std::string & name, const std::string& ticker, const std::string& key) const;
        inline std::string get_string(const std::string & name, const std::string& ticker, const std::string& key) const;
      private:
        NTKD* numeric_data_;
        NTKS* string_data_;
        friend class boost::serialization::access;
        template<class Archive>
        void serialize(Archive & ar, const unsigned int version) {
          ar & numeric_data_;
          ar & string_data_;
        }
      };

     public:
      static StaticPlayer& Inst() { static StaticPlayer um; return um; }
      double query_numeric(const std::string&name, const std::string&ticker, int32_t date, const std::string& key);
      std::string query_string(const std::string&name, const std::string&ticker, int32_t date, const std::string& key);
      void operator=(const StaticPlayer&) = delete;
      DM* operator[](int32_t date);
      void load_config(const std::set<std::string> & fset);
      void load_config(bool simple = false);

     private:
      double special_query(const std::string&ticker, int32_t date, const std::string& key);
     private:
      bool loaded_ = false;
      std::map<int32_t, DM*> umap_;
      // std::mutex mut_;
      NDTKD numeric_data_; //  name : date : ticker : key : value
      NDTKS string_data_; //  name : date : ticker : key : value
      std::unordered_map<std::string, std::string> fill_na_method_;  // name : fill_na
      std::map<int32_t, std::unordered_map<std::string, std::unordered_map<int32_t, std::string> >> rank_map_;
      std::unordered_set<std::string> tickers_, chains_;

      friend class boost::serialization::access;
      template<class Archive>
      void serialize(Archive & ar, const unsigned int version) {
        ar & loaded_; ar & umap_; ar & numeric_data_;
        ar & string_data_; ar & fill_na_method_;
        ar & rank_map_; ar & tickers_; ar & chains_;
        // ar & mut_;
      }
    };
}

the main.cpp is:

#include "./static_player.h"
void write_univ() {
  auto & u = util::trade_util::StaticPlayer::Inst();
  u.load_config();
  std::ofstream ofs("a");
  boost::archive::text_oarchive oa(ofs);
  oa << u;
  
  std::ifstream ifs("a");
  boost::archive::text_iarchive ia(ifs);
  util::trade_util::StaticPlayer& u2 = (util::trade_util::StaticPlayer::Inst());
  ia >> u2; 
  for (const auto & ticker : u2.get_tickers()) cout << ticker << endl;
}

int main() {
  write_univ();
}

the error message is:

terminate called after throwing an instance of 'boost::archive::archive_exception'
  what():  input stream error
[1]    20341 abort (core dumped)  ./a.out

I have read all i can find on the Internet, but still can't figure out why.

Could you help on this?

score 0 · Answer 1 · answered Jan 03 '22 at 00:07

Oof, serializing from/into a singleton is probably not a good idea to start with. Why not make load return an instance, that you can then choose to replace the singleton with if you must?

The real problem is a classic pitfall with serialization: if you put serialization/deserialization in the same scope, make sure your archive is completed (and written to disk) before trying to load it back...:

void write_univ()
{
    {
        std::ofstream                 ofs("a");
        boost::archive::text_oarchive oa(ofs);

        auto& u = util::trade_util::StaticPlayer::Inst();
        u.load_config();
        oa << u;
    }

    {
        std::ifstream                   ifs("a");
        boost::archive::text_iarchive   ia(ifs);
        util::trade_util::StaticPlayer& u2 =
            util::trade_util::StaticPlayer::Inst();
        ia >> u2;
    }
}

That's all.

Side Notes

Also, maybe you should try to tidy up the type "spaghetti" surrounding the maps:

template <typename T> using Dict = std::unordered_map<std::string, T>;
template <typename T> using NTK  = std::unordered_map<std::string, Dict<T>>;
template <typename T> using NDTK = std::unordered_map<std::string, std::map<int32_t, Dict<T>>>;

using NTKD  = NTK<double>;
using NTKS  = NTK<std::string>;
using NDTKD = NDTK<double>;
using NDTKS = NDTK<std::string>;

And this then tells me you're basically having straight records like

struct Record {
     std::string name;
     int32_t id;
     variant<double, string> value;
};

Why don't you consider using a single Multi-Index container with two indexes - or perhaps even just the one composed-key index by (name,id) that you can query with a partial key (id unspecified) if you prefer?

See e.g. this similar example Using boost multi index like relational DB

Self-Contained Test:

~~Live~~Undead On Coliru

// static_player.h
//#pragma once
#include <boost/archive/text_iarchive.hpp>
#include <boost/archive/text_oarchive.hpp>
#include <boost/serialization/map.hpp>
#include <boost/serialization/unordered_map.hpp>
#include <boost/serialization/unordered_set.hpp>
#include <boost/serialization/vector.hpp>
#include <fstream>
#include <memory>
#include <set>
#include <vector>

//#include "util/file_util.h"
//#include "struct/struct_util.hpp"

namespace util::trade_util {
    template <typename T> using Dict = std::unordered_map<std::string, T>;
    template <typename T> using NTK  = std::unordered_map<std::string, Dict<T>>;
    template <typename T> using NDTK = std::unordered_map<std::string, std::map<int32_t, Dict<T>>>;

    using NTKD  = NTK<double>;
    using NTKS  = NTK<std::string>;
    using NDTKD = NDTK<double>;
    using NDTKS = NDTK<std::string>;

    class StaticPlayer {
      private:
        StaticPlayer()                     = default;
        StaticPlayer(const StaticPlayer&)  = delete;
        StaticPlayer(const StaticPlayer&&) = delete;
        virtual ~StaticPlayer()            = default;

        class DM {
          public:
            DM() = default;
            DM(NTKD* numeric_data, NTKS* string_data)
                : numeric_data_(numeric_data)
                , string_data_(string_data)
            {
            }
            virtual ~DM() = default;
            [[nodiscard]] inline double
            get_numeric(const std::string& name, const std::string& ticker,
                        const std::string& key) const;
            [[nodiscard]] inline std::string
            get_string(const std::string& name, const std::string& ticker,
                       const std::string& key) const;

          private:
            NTKD* numeric_data_ = nullptr;
            NTKS* string_data_  = nullptr;
            friend class boost::serialization::access;
            template <class Archive>
            void serialize(Archive& ar, const unsigned int version)
            {
                ar& numeric_data_;
                ar& string_data_;
            }
        };

      public:
        static StaticPlayer& Inst()
        {
            static StaticPlayer um;
            return um;
        }
        double query_numeric(const std::string& name, const std::string& ticker,
                             int32_t date, const std::string& key);
        std::string query_string(const std::string& name,
                                 const std::string& ticker, int32_t date,
                                 const std::string& key);
        void        operator=(const StaticPlayer&) = delete;
        DM*         operator[](int32_t date);
        void        load_config(const std::set<std::string>& fset);
        void        load_config(bool simple = false) {}

      private:
        double special_query(const std::string& ticker, int32_t date,
                             const std::string& key);

      
        bool                   loaded_ = false;
        std::map<int32_t, DM*> umap_;
        // std::mutex mut_;
        NDTKD numeric_data_; //  name : date : ticker : key : value
        NDTKS string_data_;  //  name : date : ticker : key : value
        std::unordered_map<std::string, std::string>
            fill_na_method_; // name : fill_na
        std::map<int32_t,
                 std::unordered_map<std::string,
                                    std::unordered_map<int32_t, std::string>>>
                                        rank_map_;
        std::unordered_set<std::string> tickers_, chains_;

        friend class boost::serialization::access;
        template <class Archive>
        void serialize(Archive& ar, const unsigned int version)
        {
            ar& loaded_;
            ar& umap_;
            ar& numeric_data_;
            ar& string_data_;
            ar& fill_na_method_;
            ar& rank_map_;
            ar& tickers_;
            ar& chains_;
            // ar & mut_;
        }
    };
} // namespace util::trade_util

//#include "./static_player.h"
void write_univ()
{
    {
        std::ofstream                 ofs("a");
        boost::archive::text_oarchive oa(ofs);

        auto& u = util::trade_util::StaticPlayer::Inst();
        u.load_config();
        oa << u;
    }

    {
        std::ifstream                   ifs("a");
        boost::archive::text_iarchive   ia(ifs);
        util::trade_util::StaticPlayer& u2 =
            util::trade_util::StaticPlayer::Inst();
        ia >> u2;
    }
    // for (const auto& ticker : u2.get_tickers())
    // std::cout << ticker << std::endl;
}

int main() { write_univ(); }

Which works as expected. On my machine creates an archive a containing:

22 serialization::archive 19 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 1 0

why my boost serialization failed with input stream error?

1 Answers1

Side Notes

Self-Contained Test: