AFAIK, you can't. Whatever is in front of the colon in the object literal notation will be automatically interpreted as a string. You will need to construct your object beforehand, and use the square bracket notation.
var options = {}
options[self.settings.direction] = '+=' + self.displacement + 'px';
$(selector).animate(options, "slow" , function () {});
EDIT: You can now:
const prop = "foo"
const bad = { prop: "bar" } // { prop: "bar" }
const good = { [prop]: "bar" } // { foo: "bar" }
so...
$(selector).animate({ [self.settings.direction]: '+='+self.displacement+'px'}, "slow" , function () {});