python- how to display size of all variables

Question

I want to print the memory size of all variables in my scope simultaneously.

Something similar to:

for obj in locals().values():
    print sys.getsizeof(obj)

But with variable names before each value so I can see which variables I need to delete or split into batches.

Ideas?

Answer 1

A bit more code, but works in Python 3 and gives a sorted, human readable output:

import sys
def sizeof_fmt(num, suffix='B'):
    ''' by Fred Cirera,  https://stackoverflow.com/a/1094933/1870254, modified'''
    for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
        if abs(num) < 1024.0:
            return "%3.1f %s%s" % (num, unit, suffix)
        num /= 1024.0
    return "%.1f %s%s" % (num, 'Yi', suffix)

for name, size in sorted(((name, sys.getsizeof(value)) for name, value in list(
                          locals().items())), key= lambda x: -x[1])[:10]:
    print("{:>30}: {:>8}".format(name, sizeof_fmt(size)))

Example output:

                  umis:   3.6 GiB
       barcodes_sorted:   3.6 GiB
          barcodes_idx:   3.6 GiB
              barcodes:   3.6 GiB
                  cbcs:   3.6 GiB
         reads_per_umi:   1.3 GiB
          umis_per_cbc:  59.1 MiB
         reads_per_cbc:  59.1 MiB
                   _40:  12.1 KiB
                     _:   1.6 KiB

Note, that this will only print the 10 largest variables and remains silent about the rest. If you really want all printed, remove the [:10] from the second last line.

Answer 2

You can iterate over both the key and value of a dictionary using .items()

from __future__ import print_function  # for Python2
import sys

local_vars = list(locals().items())
for var, obj in local_vars:
    print(var, sys.getsizeof(obj))

Answer 3

I found that for some containers I wasn't getting the correct answer (overheads only?).
Combining @jan_Glx's ans above to a snippet from the post below
How to know bytes size of python object like arrays and dictionaries? - The simple way

from __future__ import print_function
from sys import getsizeof, stderr, getsizeof
from itertools import chain
from collections import deque
try:
    from reprlib import repr
except ImportError:
    pass

def sizeof_fmt(num, suffix='B'):
    ''' by Fred Cirera,  https://stackoverflow.com/a/1094933/1870254, modified'''
    for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
        if abs(num) < 1024.0:
            return "%3.1f %s%s" % (num, unit, suffix)
        num /= 1024.0
    return "%.1f %s%s" % (num, 'Yi', suffix)

def total_size(o, handlers={}, verbose=False):
    """ Returns the approximate memory footprint an object and all of its contents.

    Automatically finds the contents of the following builtin containers and
    their subclasses:  tuple, list, deque, dict, set and frozenset.
    To search other containers, add handlers to iterate over their contents:

        handlers = {SomeContainerClass: iter,
                    OtherContainerClass: OtherContainerClass.get_elements}

    """
    dict_handler = lambda d: chain.from_iterable(d.items())
    all_handlers = {tuple: iter,
                    list: iter,
                    deque: iter,
                    dict: dict_handler,
                    set: iter,
                    frozenset: iter,
                   }
    all_handlers.update(handlers)     # user handlers take precedence
    seen = set()                      # track which object id's have already been seen
    default_size = getsizeof(0)       # estimate sizeof object without __sizeof__

    def sizeof(o):
        if id(o) in seen:       # do not double count the same object
            return 0
        seen.add(id(o))
        s = getsizeof(o, default_size)

        if verbose:
            print(s, type(o), repr(o), file=stderr)

        for typ, handler in all_handlers.items():
            if isinstance(o, typ):
                s += sum(map(sizeof, handler(o)))
                break
        return s

    return sizeof(o)


##### Example call #####

for name, size in sorted(((name, total_size(value, verbose=False)) for name, value in list(
                          locals().items())), key= lambda x: -x[1])[:20]:
    print("{:>30}: {:>8}".format(name, sizeof_fmt(size)))