How to add an axis divider in seaborn?

Question

The accepted answer of this question shows how add a colorbar to a seaborn plot.

The accepted answer of this question shows how to use an AxisDivider in matplotlib.

I'd like to combine both approaches and have an axis divider in seaborn. Is that possible? For example, combining the code of the two previously mentioned posts results in an error message in seaborn:

import seaborn as sns
import matplotlib.pyplot as plt
from mpl_toolkits.axes_grid1 import make_axes_locatable


tips = sns.load_dataset("tips")
ax = sns.scatterplot(x="total_bill", y="tip", hue="size", palette='RdBu', data=tips)

norm = plt.Normalize(tips['size'].min(), tips['size'].max())
sm = plt.cm.ScalarMappable(cmap="RdBu", norm=norm)
sm.set_array([])

# Remove the legend and add a colorbar
ax.get_legend().remove()
divider = make_axes_locatable(ax)
cax = divider.append_axes("right", size="5%", pad=0.05)
plt.colorbar(im, cax=cax)
plt.show()
plt.close()

Error stack trace:

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-30-ec1c1baf2a04> in <module>
     15 divider = make_axes_locatable(ax)
     16 cax = divider.append_axes("right", size="5%", pad=0.05)
---> 17 plt.colorbar(im, cax=cax)
     18 plt.show()
     19 plt.close()

/usr/local/lib/python3.6/dist-packages/matplotlib/pyplot.py in colorbar(mappable, cax, ax, **kw)
   2192     if ax is None:
   2193         ax = gca()
-> 2194     ret = gcf().colorbar(mappable, cax=cax, ax=ax, **kw)
   2195     return ret
   2196 colorbar.__doc__ = matplotlib.colorbar.colorbar_doc

/usr/local/lib/python3.6/dist-packages/matplotlib/figure.py in colorbar(self, mappable, cax, ax, use_gridspec, **kw)
   2341                              'panchor']
   2342         cb_kw = {k: v for k, v in kw.items() if k not in NON_COLORBAR_KEYS}
-> 2343         cb = cbar.colorbar_factory(cax, mappable, **cb_kw)
   2344 
   2345         self.sca(current_ax)

/usr/local/lib/python3.6/dist-packages/matplotlib/colorbar.py in colorbar_factory(cax, mappable, **kwargs)
   1732         cb = ColorbarPatch(cax, mappable, **kwargs)
   1733     else:
-> 1734         cb = Colorbar(cax, mappable, **kwargs)
   1735 
   1736     cid = mappable.callbacksSM.connect('changed', cb.update_normal)

/usr/local/lib/python3.6/dist-packages/matplotlib/colorbar.py in __init__(self, ax, mappable, **kwargs)
   1200         # Ensure the given mappable's norm has appropriate vmin and vmax set
   1201         # even if mappable.draw has not yet been called.
-> 1202         if mappable.get_array() is not None:
   1203             mappable.autoscale_None()
   1204 

AttributeError: 'AxesSubplot' object has no attribute 'get_array'

I'd appreciate some help. :)