Determine if a Selection Marquee is over a Rotated Rectangle

Question

I have a Rectangle class for drawing to HTML Canvas. It has a rotation property that gets applied in its draw method. If the user drags within the canvas, a selection marquee is being drawn. How can I set the Rectangle's active attribute to true when the Rectangle is within the selection marquee using math? This is a problem I'm having in another language & context so I do not have all of Canvas' methods available to me there (e.g. isPointInPath).

I found a StackOverflow post about finding Mouse position within rotated rectangle in HTML5 Canvas, which I am implementing in the Rectangle method checkHit. It doesn't account for the selection marquee, however. It's just looking at the mouse X & Y, which is still off. The light blue dot is the origin around which the rectangle is being rotated. Please let me know if anything is unclear. Thank you.

class Rectangle
{
  constructor(x, y, width, height, rotation) {
    this.x = x;
    this.y = y;
    this.height = height;
    this.width = width;
    this.xOffset = this.x + this.width/2;
    this.yOffset = this.y + ((this.y+this.height)/2);
    this.rotation = rotation;
    this.active = false;
  }
  
  checkHit()
  {
    // translate mouse point values to origin
    let originX = this.xOffset;
    let originY = this.yOffset;
    let dx = marquee[2] - originX;
    let dy = marquee[3] - originY;
    // distance between the point and the center of the rectangle
    let h1 = Math.sqrt(dx*dx + dy*dy);
    let currA = Math.atan2(dy,dx);
    // Angle of point rotated around origin of rectangle in opposition
    let newA = currA - this.rotation;
    // New position of mouse point when rotated
    let x2 = Math.cos(newA) * h1;
    let y2 = Math.sin(newA) * h1;
    // Check relative to center of rectangle
    if (x2 > -0.5 * this.width && x2 < 0.5 * this.width && y2 > -0.5 * this.height && y2 < 0.5 * this.height){
      this.active = true;
    } else {
      this.active = false;    
    }
    
  }
  
  draw()
  {
    ctx.save();
    ctx.translate(this.xOffset, this.yOffset);
    ctx.fillStyle = 'rgba(255,255,255,1)';
    ctx.beginPath();
    ctx.arc(0, 0, 3, 0, 2 * Math.PI, true);
    ctx.fill();
    ctx.rotate(this.rotation * Math.PI / 180);
    ctx.translate(-this.xOffset, -this.yOffset);
    if (this.active)
    {
      ctx.fillStyle = 'rgba(255,0,0,0.5)';
    } else {
      ctx.fillStyle = 'rgba(0,0,255,0.5)';      
    }
    ctx.beginPath();
    ctx.fillRect(this.x, this.y, this.width, this.y+this.height);
    ctx.closePath();
    ctx.stroke();
    ctx.restore();
  }
}

var canvas = document.getElementById("canvas");
var ctx = canvas.getContext("2d");
var raf;
var rect = new Rectangle(50,50,90,30,45);
var marquee = [-3,-3,-3,-3];
var BB=canvas.getBoundingClientRect();
var offsetX=BB.left;
var offsetY=BB.top;
var start_x,start_y;

let draw = () => {
  ctx.clearRect(0,0, canvas.width, canvas.height);
  //rect.rotation+=1;
  rect.draw();
  ctx.fillStyle = "rgba(200, 200, 255, 0.5)";
  ctx.fillRect(parseInt(marquee[0]),parseInt(marquee[1]),parseInt(marquee[2]),parseInt(marquee[3]))
  ctx.strokeStyle = "white"
  ctx.lineWidth = 1;
  ctx.rect(parseInt(marquee[0]),parseInt(marquee[1]),parseInt(marquee[2]),parseInt(marquee[3]))
  ctx.stroke()
  raf = window.requestAnimationFrame(draw);
}

let dragStart = (e) =>
{
  start_x = parseInt(e.clientX-offsetX);
  start_y = parseInt(e.clientY-offsetY);
  marquee = [start_x,start_y,0,0];
  canvas.addEventListener("mousemove", drag);
}

let drag = (e) =>
{
  let mouseX = parseInt(e.clientX-offsetX);
  let mouseY = parseInt(e.clientY-offsetY);
  marquee[2] = mouseX - start_x;
    marquee[3] = mouseY - start_y;
  rect.checkHit();
}

let dragEnd = (e) =>
{
  marquee = [-10,-10,-10,-10];
  canvas.removeEventListener("mousemove", drag);
}

canvas.addEventListener('mousedown', dragStart);
canvas.addEventListener('mouseup', dragEnd);

raf = window.requestAnimationFrame(draw);

body
{
  margin:0;  
}

#canvas
{
  width: 360px;
  height: 180px;
  border: 1px solid grey;
  background-color: grey;
}

<canvas id="canvas" width="360" height="180"></canvas>

Answer 1

Do convex polygons overlap

Rectangles are convex polygons.

Rectangle and marquee each have 4 points (corners) that define 4 edges (lines segments) connecting the points.

This solution works for all convex irregular polygons with 3 or more sides.

Points and edges must be sequential either Clockwise CW of Count Clockwise CCW

Test points

If any point of one poly is inside the other polygon then they must overlap. See example function isInside

To check if point is inside a polygon, get cross product of, edge start to the point as vector, and the edge as a vector.

If all cross products are >= 0 (to the left of) then there is overlap (for CW polygon). If polygon is CCW then if all cross products are <= 0 (to the right of) there is overlap.

It is possible to overlap without any points inside the other poly.

Test Edges

If any of the edges from one poly crosses any of the edges from the other then there must be overlap. The function doLinesIntercept returns true if two line segments intercept.

Complete test

Function isPolyOver(poly1, poly2) will return true if there is overlap of the two polys.

A polygon is defined by a set of Point's and Lines's connecting the points.

The polygon can be irregular, meaning that each edge can be any length > 0

Do not pass polygons with an edge of length === 0 or will not work.

Added

I added the function Rectangle.toPoints that transforms the rectangle and returning a set of 4 points (corners).

Example

Example is a copy of your code working using the above methods.

canvas.addEventListener('mousedown', dragStart);
canvas.addEventListener('mouseup', dragEnd);
requestAnimationFrame(draw);

const Point = (x = 0, y = 0) => ({x, y, set(x,y){ this.x = x; this.y = y }});
const Line = (p1, p2) => ({p1, p2});
const selector = { points: [Point(), Point(), Point(), Point()] }
selector.lines = [
    Line(selector.points[0], selector.points[1]),
    Line(selector.points[1], selector.points[2]),
    Line(selector.points[2], selector.points[3]),
    Line(selector.points[3], selector.points[0])
];
const rectangle = { points: [Point(), Point(), Point(), Point()] }
rectangle.lines = [
    Line(rectangle.points[0], rectangle.points[1]),
    Line(rectangle.points[1], rectangle.points[2]),
    Line(rectangle.points[2], rectangle.points[3]),
    Line(rectangle.points[3], rectangle.points[0])
];

function isInside(point, points) {
    var i = 0, p1 = points[points.length - 1];
    while (i < points.length) {
        const p2 = points[i++];
        if ((p2.x - p1.x) * (point.y - p1.y) - (p2.y - p1.y) * (point.x - p1.x) < 0) { return false }
        p1 = p2;
    }
    return true;
}
function doLinesIntercept(l1, l2) { 
    const v1x = l1.p2.x - l1.p1.x;
    const v1y = l1.p2.y - l1.p1.y;
    const v2x = l2.p2.x - l2.p1.x;
    const v2y = l2.p2.y - l2.p1.y;
    const c = v1x * v2y - v1y * v2x;
    if(c !== 0){
        const u = (v2x * (l1.p1.y - l2.p1.y) - v2y * (l1.p1.x - l2.p1.x)) / c;
        if(u >= 0 && u <= 1){
            const u = (v1x * (l1.p1.y - l2.p1.y) - v1y * (l1.p1.x - l2.p1.x)) / c;
            return  u >= 0 && u <= 1;
        }
    }
    return false;
}   
function isPolyOver(p1, p2) { // is poly p2 under any part of poly p1
    if (p2.points.some(p => isInside(p, p1.points))) { return true };
    if (p1.points.some(p => isInside(p, p2.points))) { return true };
    return p1.lines.some(l1 => p2.lines.some(l2 => doLinesIntercept(l1, l2)));
}
    
const ctx = canvas.getContext("2d");
var dragging = false;

const marquee = [0,0,0,0];
const rotate = 0.01;
var startX, startY, hasSize = false;
const BB = canvas.getBoundingClientRect();
const offsetX = BB.left;
const offsetY = BB.top;
class Rectangle {
    constructor(x, y, width, height, rotation) {
        this.x = x;
        this.y = y;
        this.height = height;
        this.width = width;
        this.rotation = rotation;
        this.active = false;
    }
    toPoints(points = [Point(), Point(), Point(), Point()]) {
        const xAx = Math.cos(this.rotation) / 2;
        const xAy = Math.sin(this.rotation) / 2;
        const x = this.x, y = this.y;
        const w = this.width, h = this.height;
        points[0].set(-w * xAx + h * xAy + x, -w * xAy - h * xAx + y);
        points[1].set( w * xAx + h * xAy + x,  w * xAy - h * xAx + y);
        points[2].set( w * xAx - h * xAy + x,  w * xAy + h * xAx + y);
        points[3].set(-w * xAx - h * xAy + x, -w * xAy + h * xAx + y);
    }
    draw() {
        ctx.setTransform(1, 0, 0, 1, this.x, this.y);
        ctx.fillStyle = 'rgba(255,255,255,1)';
        ctx.strokeStyle = this.active ? 'rgba(255,0,0,1)' : 'rgba(0,0,255,1)';
        ctx.lineWidth = this.active ? 3 : 1;
        
        ctx.beginPath();
        ctx.arc(0, 0, 3, 0, 2 * Math.PI, true);
        ctx.fill();
        ctx.rotate(this.rotation);
        
        ctx.beginPath();
        ctx.rect(-this.width / 2, - this.height / 2, this.width, this.height);
        ctx.stroke();
    }
}
function draw(){
    rect.rotation += rotate;
    ctx.setTransform(1, 0, 0, 1, 0, 0);
    ctx.clearRect(0, 0, canvas.width, canvas.height);
    rect.draw();
    drawSelector();
    requestAnimationFrame(draw);
}
function drawSelector() {
    if (dragging && hasSize) {
        rect.toPoints(rectangle.points);
        rect.active = isPolyOver(selector, rectangle);
        ctx.setTransform(1, 0, 0, 1, 0, 0);
        ctx.fillStyle = "rgba(200, 200, 255, 0.5)";
        ctx.strokeStyle = "white";
        ctx.lineWidth = 1;
        ctx.beginPath();
        ctx.rect(...marquee);
        ctx.fill();
        ctx.stroke();
    
    } else {
        rect.active = false;
    }

 
}
function dragStart(e) {
    startX = e.clientX - offsetX;
    startY = e.clientY - offsetY;
    drag(e);
    canvas.addEventListener("mousemove", drag);
    
}
function drag(e) {
    dragging = true;
    const x = e.clientX - offsetX;
    const y = e.clientY - offsetY;
    const left = Math.min(startX, x);
    const top = Math.min(startY, y);
    const w = Math.max(startX, x) - left;
    const h = Math.max(startY, y) - top;
    marquee[0] = left;
    marquee[1] = top;
    marquee[2] = w;
    marquee[3] = h;
    if (w > 0 || h > 0) {
        hasSize = true;
        selector.points[0].set(left,   top);
        selector.points[1].set(left + w, top);
        selector.points[2].set(left + w, top + h);
        selector.points[3].set(left  , top + h);
        
    } else {
        hasSize = false;
    }
}
function dragEnd(e) {
    dragging = false;
    rect.active = false;
    canvas.removeEventListener("mousemove", drag);
}

const rect = new Rectangle(canvas.width / 2, canvas.height / 2, 90, 90, Math.PI / 4);

body
{
  margin:0;  
}

#canvas
{
  width: 360px;
  height: 180px;
  border: 1px solid grey;
  background-color: grey;
}

<canvas id="canvas" width="360" height="180"></canvas>