Alright, let's have a go at this! We'll have to order these objects not by distance, but by obstruction. By that, I mean for two objects A and B, A can visibly obstruct B, B can visibly obstruct A, or neither obstructs the other. If A obstructs B, we'll want to draw B first, and vice-versa. To solve this, we need to be able to say whether A obstructs B, or the other way around.
Here's what I've come up with. I only had a limited ability to test this, so there may still be flaws, but the thought process is sound.
Step 1. Map each object to its bounds, saving the original object for later:
let step1 = objects.map(o => ({
original: o,
xmin: o.x,
xmax: o.x + o.w,
ymin: o.y,
ymax: o.y + o.h
}));
Step 2. Map each object to the two corners that, when a line is drawn between them, form the largest obstruction to the camera's field of view:
let step2 = step1.map(o => {
const [closestX, farthestX] = [o.xmin, o.xmax].sort((a, b) => Math.abs(camera.x - a) - Math.abs(camera.x - b));
const [closestY, farthestY] = [o.ymin, o.ymax].sort((a, b) => Math.abs(camera.y - a) - Math.abs(camera.y - b));
return {
original: o.original,
x1: closestX,
y1: o.xmin <= camera.x && camera.x <= o.xmax ? closestY : farthestY,
x2: o.ymin <= camera.y && camera.y <= o.ymax ? closestX : farthestX,
y2: closestY
};
});
Step 3. Sort the objects. Draw a line segment from the camera to each endpoint of one object. If the line segment between the endpoints of the other object intersects, the other object is closer and must be drawn after.
let step3 = step2.sort((a, b) => {
const camSegmentA1 = {
x1: camera.x,
y1: camera.y,
x2: a.x1,
y2: a.y1
};
const camSegmentA2 = {
x1: camera.x,
y1: camera.y,
x2: a.x2,
y2: a.y2
};
const camSegmentB1 = {
x1: camera.x,
y1: camera.y,
x2: b.x1,
y2: b.y1
};
const camSegmentB2 = {
x1: camera.x,
y1: camera.y,
x2: b.x2,
y2: b.y2
};
// Intersection function taken from here: https://stackoverflow.com/a/24392281
function intersects(seg1, seg2) {
const a = seg1.x1, b = seg1.y1, c = seg1.x2, d = seg1.y2,
p = seg2.x1, q = seg2.y1, r = seg2.x2, s = seg2.y2;
const det = (c - a) * (s - q) - (r - p) * (d - b);
if (det === 0) {
return false;
} else {
lambda = ((s - q) * (r - a) + (p - r) * (s - b)) / det;
gamma = ((b - d) * (r - a) + (c - a) * (s - b)) / det;
return (0 < lambda && lambda < 1) && (0 < gamma && gamma < 1);
}
}
function squaredDistance(pointA, pointB) {
return Math.pow(pointB.x - pointA.x, 2) + Math.pow(pointB.y - pointA.y, 2);
}
if (intersects(camSegmentA1, b) || intersects(camSegmentA2, b)) {
return -1;
} else if (intersects(camSegmentB1, a) || intersects(camSegmentB2, a)) {
return 1;
} else {
return Math.max(squaredDistance(camera, {x: b.x1, y: b.y1}), squaredDistance(camera, {x: b.x2, y: b.y2})) - Math.max(squaredDistance(camera, {x: a.x1, y: a.y1}), squaredDistance(camera, {x: a.x2, y: a.y2}));
}
});
Step 4. Final step -- get the original objects back, sorted in order of farthest to closest:
let results = step3.map(o => o.original);
Now, to put it all together:
results = objects.map(o => {
const xmin = o.x,
xmax = o.x + o.w,
ymin = o.y,
ymax = o.y + o.h;
const [closestX, farthestX] = [xmin, xmax].sort((a, b) => Math.abs(camera.x - a) - Math.abs(camera.x - b));
const [closestY, farthestY] = [ymin, ymax].sort((a, b) => Math.abs(camera.y - a) - Math.abs(camera.y - b));
return {
original: o,
x1: closestX,
y1: xmin <= camera.x && camera.x <= xmax ? closestY : farthestY,
x2: ymin <= camera.y && camera.y <= ymax ? closestX : farthestX,
y2: closestY
};
}).sort((a, b) => {
const camSegmentA1 = {
x1: camera.x,
y1: camera.y,
x2: a.x1,
y2: a.y1
};
const camSegmentA2 = {
x1: camera.x,
y1: camera.y,
x2: a.x2,
y2: a.y2
};
const camSegmentB1 = {
x1: camera.x,
y1: camera.y,
x2: b.x1,
y2: b.y1
};
const camSegmentB2 = {
x1: camera.x,
y1: camera.y,
x2: b.x2,
y2: b.y2
};
// Intersection function taken from here: https://stackoverflow.com/a/24392281
function intersects(seg1, seg2) {
const a = seg1.x1, b = seg1.y1, c = seg1.x2, d = seg1.y2,
p = seg2.x1, q = seg2.y1, r = seg2.x2, s = seg2.y2;
const det = (c - a) * (s - q) - (r - p) * (d - b);
if (det === 0) {
return false;
} else {
lambda = ((s - q) * (r - a) + (p - r) * (s - b)) / det;
gamma = ((b - d) * (r - a) + (c - a) * (s - b)) / det;
return (0 < lambda && lambda < 1) && (0 < gamma && gamma < 1);
}
}
function squaredDistance(pointA, pointB) {
return Math.pow(pointB.x - pointA.x, 2) + Math.pow(pointB.y - pointA.y, 2);
}
if (intersects(camSegmentA1, b) || intersects(camSegmentA2, b)) {
return -1;
} else if (intersects(camSegmentB1, a) || intersects(camSegmentB2, a)) {
return 1;
}
return Math.max(squaredDistance(camera, {x: b.x1, y: b.y1}), squaredDistance(camera, {x: b.x2, y: b.y2})) - Math.max(squaredDistance(camera, {x: a.x1, y: a.y1}), squaredDistance(camera, {x: a.x2, y: a.y2}));
}).map(o => o.original);
Let me know if that works!