I cannot see how to do this in a complex case, where the selection contains several HTML elements that might be setting color and background color individually (though of course some complex HTML/CSS parsing must be able to do that but I'm not capable).
However, we can fairly simply deal with the straightforward case where the color and background color are consistent across the parent div of the selection.
On a selection event this snippet reads the computed style of the parent element, calculates the inverse of the color and backgroundColor and sets CSS variables to these which are picked up and used in the selection pseudo element.
document.addEventListener('selectionchange', (e) => {
function invert(color) {
color = color.replace('rgb(', '');
color = color.replace(')', '');
color = color.replace('rgba(', '');
let colors = color.split(',');
let r = colors[0].replace(' ', '');
let g = colors[1].replace(' ', '');
let b = colors[2].replace(' ', '');
let a = (colors.length > 3) ? colors[3].replace(' ', '') : 1;
r = ~r & 255;
g = ~g & 255;
b = ~b & 255;
return 'rgba(' + r + ', ' + g + ', ' + b + ', ' + a + ')';
}
selection = window.getSelection();
if (selection.anchorNode) {
const parent = selection.anchorNode.parentElement;
const style = window.getComputedStyle(parent);
parent.style.setProperty('--color', invert(style.color));
parent.style.setProperty('--background-color', invert(style.backgroundColor));
}
});
.invertSelectionColors::selection {
color: var(--color);
background-color: var(--background-color);
}
<div class="invertSelectionColors" style="background-color: cyan;color:#000000;">
here is some text originally background color cyan and color #000000
</div>
<div class="invertSelectionColors" style="background-color: rgba(255, 0, 0, 0.5);color:#0000ffff;">
here is some text originally background color rgba(255, 0, 0, 0.5) and color #0000ffff
</div>