0

edit: The below code can be run on https://wandbox.org/permlink/1Qry83quzoDveYDi
I implemented various suggestions from the comments, but unfortunately, it is still not clear to my why erasing the item at this particular std::list::iterator (line 86) crashes at runtime. All the explanations given below seem to affirm that the iterator should still be valid at this point.


I was under the impression that iterators to items in a std::list do not get invalidated when an item is inserted into a list (refer to this excellent post).

However, in the below code, the line
items.at(noOfitems-2)->erase(iter++); (line 86)
crashes the program with
malloc: *** error for object 0x100778b28: pointer being freed was not allocated.

Could you please help me understand why (where) this iterator std::list<std::string>::iterator becomes invalid, and how I can make it work without iteratively finding it again?

Am I perhaps misunderstanding the error?

#include <iostream>
#include <iostream>
#include <vector>
#include <random>
#include <unordered_set>
#include <set>
#include <unordered_map>
#include <map>
#include <list>
#include <utility>
#include <chrono>
#include <sstream>
#include <tr1/memory>

class Increment;
struct Item;

struct Pairhash {
public:
  template <typename T>
  std::size_t operator()(const T &x) const
  {
      return std::hash<T>()(x) ^ std::hash<T>()(x);
  }
};

struct Item {
    Item() = default;
    std::string name;
    int counter;
    Item(std::string name) : counter(0)
    {
        this->name = name;
    }
    
    bool operator==(const Item& p) const
    {
        return (this->name == p.name);
    }
    
    bool operator<(const Item& p) const
    {
        return (this->counter < p.counter);
    }
};

class Increment {
private:
    std::unordered_map<std::string, std::pair<std::list<std::string>::iterator , std::shared_ptr<Item> >, Pairhash >  itemMap;
    std::vector<std::shared_ptr<std::list<std::string>>> items;
public:
    Increment() = default;
    std::list<std::string>::iterator insertItem(std::string & name , int noOfitems)
    {
        if (noOfitems > items.size())
        {
            items.emplace_back(std::make_shared<std::list<std::string>>(std::initializer_list<std::string>{ name }));
            return items.back()->begin();
        }
        else
        {
            items.at(noOfitems-1)->emplace_back(name);
            return items.at(noOfitems-1)->rbegin().base(); //Last position in list
        }
    }
    
    std::list<std::string>::iterator adjustItemPosition(std::string & name, int noOfitems, std::list<std::string>::iterator & iter)
    {
        if (noOfitems > items.size())
        {
            std::list<std::string> temp{name};
            items.push_back(std::make_shared<std::list<std::string>>(temp));
        }
        else
        {
            items.at(noOfitems-1)->emplace_back(name);
        }
        /* // Works as expected
        auto itr = std::find(items.at(noOfitems-2)->begin(), items.at(noOfitems-2)->end(), name);
        if (itr != items.at(noOfitems-2)->end())
        {
            items.at(noOfitems-2)->erase(itr);
        }
        */
        items.at(noOfitems-2)->erase(iter++); //TODO Crashes
        return items.at(noOfitems-1)->rbegin().base(); //Last position in list
    }
    
    void incrementByOne(std::string name)
    {
        auto it = itemMap.find(name);
        if (it != itemMap.end()) //Item already in map
        {
            it->second.second->counter++;
            it->second.first = adjustItemPosition(name, it->second.second->counter,
                                                    it->second.first);
        }
        else  //New item to be inserted
        {
            std::shared_ptr<Item> temp = std::make_shared<Item>(Item(name));
            temp->counter = 1;
            std::list<std::string>::iterator listIter = insertItem(name, 1);
            itemMap.emplace(name, std::make_pair( listIter, temp));
        }
    }
    
    std::string printTop10() const
    {
        std::stringstream ss;
        auto count(0);
        for (auto it = items.rbegin(); it != items.rend(); ++it)
        {
            for (auto item : **it)
            {
                if (count == 10)
                {
                    break;
                }
                ss << "We have " << itemMap.at(item).second->counter << " " << item << std::endl;
                ++count;
            }
        }
        return ss.str();
    }
};

int main(int argc, const char * argv[]) {
    Increment incrementer;
    std::vector<std::string> names{ "Bananas", "Apples", "Peaches", "Durians", "Hazelnuts", "Avocados", "Pineapples", "Cherries", "Almonds", "Olives", "Eggs", "Yoghurts", "Peas", "Blueberries" };

    for (int i = 0; i < 100; ++i) {
        incrementer.incrementByOne(names.at(i%10));
    }
    std::cout << incrementer.printTop10() << std::endl;
    return 0;
}
TTT
  • 9
  • 3
  • 2
    Have you checked that `items.at(noOfitems-2)` is not `nullptr`? Otherwise, the only way an `std::list` iterator is invalidated is if that element no longer exists. That is one of the only concrete advantages an `std::list` has over most other sequential containers, its iterator and reference stability. – François Andrieux Aug 13 '20 at 20:50
  • Yes, I thought of that; lines 80 - 84 work when used instead of line 86. This also uses noOfItems -2. – TTT Aug 13 '20 at 20:55
  • Well, it would be Undefined Behavior if the pointer was `nullptr`, so the fact that they don't visibly fail doesn't prove that it isn't `nullptr`. Use a debugger to make sure, or print something to `std::cout` that shows that it isn't `nullptr`. – François Andrieux Aug 13 '20 at 20:58

3 Answers3

2

What causes the stored std::list::iterator to become invalid?

Removal of the element pointed by the iterator, whether through clear, pop_X, erase or destruction of the list. assign invalidates iterators to all elements of that list.

eerorika
  • 232,697
  • 12
  • 197
  • 326
  • According to the linked post, this is not the case. Quoting from https://stackoverflow.com/questions/6438086/iterator-invalidation-rules: "list: Invalidates only the iterators and references to the erased elements. [26.3.10.4/3]. This applies to erase, pop_front, pop_back, clear functions." – TTT Aug 13 '20 at 20:54
  • @TTT that does not contradict with my answer. Pay attention in particular to *"only the iterators and references **to the erased elements**"* – eerorika Aug 13 '20 at 20:55
  • I am surprised to see two comments challenging this correct answer. – SergeyA Aug 13 '20 at 20:56
  • I'm not trying to access an erased element. Lines 80-84 prove that the element exist in the expected place. – TTT Aug 13 '20 at 20:57
  • @TTT Regardless, this is the list of operations that invalidate list iterators. So, we can conclude: You are mistaken and you either 1. do access an erased element or 2. are not using an invalid iterator. – eerorika Aug 13 '20 at 20:58
  • @TTT As my comment in the question says, if the pointer was `nullptr` or if the iterator was invalidated, the result would be Undefined Behavior. So the fact that you observe them to work does not prove that they do, or that the pointer or iterator are actually valid. – François Andrieux Aug 13 '20 at 20:59
  • @cigien Re-reading your answer I apologize. Unfortunately, this doesn't solve my problem - something else must be going on here. I wondered if it could be that the std::list is held in an std::vector, but I would assume (?) that using a shared pointer to the items in the list would preserve the validity, even if the location of the vector gets updated? – TTT Aug 13 '20 at 21:01
  • @TTT Even moving an `std::list` to another keeps the validity of the iterators, they just refer to elements in a different container now. [Link](https://en.cppreference.com/w/cpp/container/list/operator%3D#Notes). – François Andrieux Aug 13 '20 at 21:02
  • @eerorika, the above code is both reproducible and complete (you can paste and run) - almost the only thing I could do to minimize it more is removing the output to std::cout, not sure that would be so much more beneficial. – TTT Aug 13 '20 at 21:05
  • @TTT Are you saying that the problem doesn't reproduce if you remove the random number generator? – eerorika Aug 13 '20 at 21:06
  • @eerorika, point taken. I will try to improve on this next time. – TTT Aug 13 '20 at 21:10
  • Have removed random number generator (@eerorika), and fixed problem highlighted by @SergeyA. Still crashes with line 86 as above, still works when using lines 80-84 instead. – TTT Aug 13 '20 at 21:18
  • @TTT — `erase(iter++)` invalidates `iter`. Incrementing `iter` doesn’t fix it. – Pete Becker Aug 13 '20 at 21:52
  • 2
    @PeteBecker But the behavior of `iter++` is to make a copy of the iterator, increment the original, then evaluate as the original (before it was incremented). So `iter` should be pointed at the next element before the prior value is used to erase, no? I don't see any reason this would invalidate `iter` since it is incremented to point at something else before the erase happens. – cdhowie Aug 13 '20 at 22:09
  • @PeteBecker erase(iter++) is not the issue and is correct syntax. The problem is reproducible using erase(iter), and I’d like to point out again that the same procedure using the commented code in lines 80-84 succeeds. Going back to @eerorika’s answer, this iterator should be valid, as it has not been cleared, popped, erased, or the list assigned at the point of access. – TTT Aug 14 '20 at 13:20
2

You have at least one issue in your code:

std::list<std::string> temp;
temp.push_back(name);
items.emplace_back(std::make_shared<std::list<std::string>>(temp));
return temp.begin();

Here you are returning an iterator to element in obliterated list. You might have other issues as well.

SergeyA
  • 61,605
  • 5
  • 78
  • 137
  • Thank you SergeyA - that is something I have not considered. Let me try fix this. – TTT Aug 13 '20 at 21:06
  • Unfortunately, replacing `return temp.begin();` with `return items.back()->begin();` yields the same crash. – TTT Aug 13 '20 at 21:09
  • @TTT This is just an optimization tip: since `temp` is doomed when the function returns, you could just steal its contents instead of needlessly copying it into a new list. `items.emplace_back(std::make_shared>(std::move(temp)));`. Alternatively, remove `temp` and construct the list in one step, in the right place: `items.emplace_back(std::make_shared>>({ name }));` – cdhowie Aug 14 '20 at 03:03
  • Hi @cdhowie, thanks for the tip. `std::move` works and is an obvious improvement. I couldn't get constructing in-place with `items.emplace_back(std::make_shared>>({ name }));` to work (doesn't compile) – TTT Aug 14 '20 at 19:47
  • @TTT My bad, you have to explicitly specify that the argument is an initializer list. The compiler can't deduce that. `items.emplace_back(std::make_shared>(std::initializer_list{ foo }));` – cdhowie Aug 14 '20 at 20:06
  • Nice, thanks @cdhowie, I was looking for this! Doesn't solve my original problem, but I've learned something here! – TTT Aug 14 '20 at 20:09
0

Replacing
return items.at(noOfitems-1)->rbegin().base();
with
return std::next(items.at(noOfitems-1)->end(), -1);

in lines 63 and 86 fixes the crash.
Working solution: https://wandbox.org/permlink/I68Szb0XMRKsPZqp

#include <iostream>
#include <iostream>
#include <vector>
#include <random>
#include <unordered_set>
#include <set>
#include <unordered_map>
#include <map>
#include <list>
#include <utility>
#include <chrono>
#include <sstream>
#include <memory>

class Increment;
struct Item;

struct Pairhash {
public:
  template <typename T>
  std::size_t operator()(const T &x) const
  {
      return std::hash<T>()(x) ^ std::hash<T>()(x);
  }
};

struct Item {
    Item() = default;
    std::string name;
    int counter;
    Item(std::string name) : counter(0)
    {
        this->name = name;
    }

    bool operator==(const Item& p) const
    {
        return (this->name == p.name);
    }

    bool operator<(const Item& p) const
    {
        return (this->counter < p.counter);
    }
};

class Increment {
private:
    std::unordered_map<std::string, std::pair<std::list<std::string>::iterator , std::shared_ptr<Item> >, Pairhash >  itemMap;
    std::vector<std::shared_ptr<std::list<std::string>>> items;
public:
    Increment() = default;
    std::list<std::string>::iterator insertItem(std::string & name , int noOfitems)
    {
        if (noOfitems > items.size())
        {
            items.emplace_back(std::make_shared<std::list<std::string>>(std::initializer_list<std::string>{ name }));
            return items.back()->begin();
        }
        else
        {
            items.at(noOfitems-1)->emplace_back(name);
            return std::next(items.at(noOfitems-1)->end(), -1);  // versus return items.at(noOfitems-1)->rbegin().base(); //Crashes
        }
    }

    std::list<std::string>::iterator adjustItemPosition(std::string & name, int noOfitems, std::list<std::string>::iterator & iter)
    {
        if (noOfitems > items.size())
        {
            std::list<std::string> temp{name};
            items.push_back(std::make_shared<std::list<std::string>>(temp));
        }
        else
        {
            items.at(noOfitems-1)->emplace_back(name);
        }
        /* // Works as expected
        auto itr = std::find(items.at(noOfitems-2)->begin(), items.at(noOfitems-2)->end(), name);
        if (itr != items.at(noOfitems-2)->end())
        {
            items.at(noOfitems-2)->erase(itr);
        }
        */
        items.at(noOfitems-2)->erase(iter++); //TODO Crashes
        return std::next(items.at(noOfitems-1)->end(), -1); //versus return items.at(noOfitems-1)->rbegin().base(); //Crashes
    }

    void incrementByOne(std::string name)
    {
        auto it = itemMap.find(name);
        if (it != itemMap.end()) //Item already in map
        {
            it->second.second->counter++;
            it->second.first = adjustItemPosition(name, it->second.second->counter,
                                                    it->second.first);
        }
        else  //New item to be inserted
        {
            std::shared_ptr<Item> temp = std::make_shared<Item>(Item(name));
            temp->counter = 1;
            std::list<std::string>::iterator listIter = insertItem(name, 1);
            itemMap.emplace(name, std::make_pair( listIter, temp));
        }
    }

    std::string printTop10() const
    {
        std::stringstream ss;
        auto count(0);
        for (auto it = items.rbegin(); it != items.rend(); ++it)
        {
            for (auto item : **it)
            {
                if (count == 10)
                {
                    break;
                }
                ss << "We have " << itemMap.at(item).second->counter << " " << item << std::endl;
                ++count;
            }
        }
        return ss.str();
    }
};

int main(int argc, const char * argv[]) {
    Increment incrementer;
    std::vector<std::string> names{ "Bananas", "Apples", "Peaches", "Durians", "Hazelnuts", "Avocados", "Pineapples", "Cherries", "Almonds", "Olives", "Eggs", "Yoghurts", "Peas", "Blueberries" };

    for (int i = 0; i < 100; ++i) {
        incrementer.incrementByOne(names.at(i%10));
    }
    std::cout << incrementer.printTop10() << std::endl;
    return 0;
}

I believe the answer as to why this is so was given in https://stackoverflow.com/a/33851951, namely that the pointer to the reverse iterator is in fact a copy of the original reference to it, which is deleted when it goes out of scope. http://cplusplus.github.io/LWG/lwg-defects.html#2360

TTT
  • 9
  • 3