I managed to get it working ...
so I did a small C++/OpenGL example where I test this:
//---------------------------------------------------------------------------
//--- input data ------------------------------------------------------------
//---------------------------------------------------------------------------
double m[16]= // matrix
{
1.0825317547305484,-0.6249999999999999,0.0,20.0,
0.6249999999999999, 1.0825317547305484,0.0,30.0,
0.0 , 0.0 ,1.0,0.0 ,
0.0 , 0.0 ,0.0,1.0
};
double a[4][3]= // your un-transformed rectangle
{
{ 30 ,50 ,0 },
{ 30+200,50 ,0 },
{ 30+200,50+100,0 },
{ 30 ,50+100,0 },
};
//---------------------------------------------------------------------------
//--- just matrix and vector math you can ignore this -----------------------
//---------------------------------------------------------------------------
void vector_copy (double *c,double *a) { for(int i=0;i<3;i++) c[i]=a[i]; }
void matrix_mul (double *c,double *a,double *b) // c[16] = a[16]*b[16]
{
double q[16];
q[ 0]=(a[ 0]*b[ 0])+(a[ 1]*b[ 4])+(a[ 2]*b[ 8])+(a[ 3]*b[12]);
q[ 1]=(a[ 0]*b[ 1])+(a[ 1]*b[ 5])+(a[ 2]*b[ 9])+(a[ 3]*b[13]);
q[ 2]=(a[ 0]*b[ 2])+(a[ 1]*b[ 6])+(a[ 2]*b[10])+(a[ 3]*b[14]);
q[ 3]=(a[ 0]*b[ 3])+(a[ 1]*b[ 7])+(a[ 2]*b[11])+(a[ 3]*b[15]);
q[ 4]=(a[ 4]*b[ 0])+(a[ 5]*b[ 4])+(a[ 6]*b[ 8])+(a[ 7]*b[12]);
q[ 5]=(a[ 4]*b[ 1])+(a[ 5]*b[ 5])+(a[ 6]*b[ 9])+(a[ 7]*b[13]);
q[ 6]=(a[ 4]*b[ 2])+(a[ 5]*b[ 6])+(a[ 6]*b[10])+(a[ 7]*b[14]);
q[ 7]=(a[ 4]*b[ 3])+(a[ 5]*b[ 7])+(a[ 6]*b[11])+(a[ 7]*b[15]);
q[ 8]=(a[ 8]*b[ 0])+(a[ 9]*b[ 4])+(a[10]*b[ 8])+(a[11]*b[12]);
q[ 9]=(a[ 8]*b[ 1])+(a[ 9]*b[ 5])+(a[10]*b[ 9])+(a[11]*b[13]);
q[10]=(a[ 8]*b[ 2])+(a[ 9]*b[ 6])+(a[10]*b[10])+(a[11]*b[14]);
q[11]=(a[ 8]*b[ 3])+(a[ 9]*b[ 7])+(a[10]*b[11])+(a[11]*b[15]);
q[12]=(a[12]*b[ 0])+(a[13]*b[ 4])+(a[14]*b[ 8])+(a[15]*b[12]);
q[13]=(a[12]*b[ 1])+(a[13]*b[ 5])+(a[14]*b[ 9])+(a[15]*b[13]);
q[14]=(a[12]*b[ 2])+(a[13]*b[ 6])+(a[14]*b[10])+(a[15]*b[14]);
q[15]=(a[12]*b[ 3])+(a[13]*b[ 7])+(a[14]*b[11])+(a[15]*b[15]);
for(int i=0;i<16;i++) c[i]=q[i];
}
void matrix_mul_vector(double *c,double *a,double *b) // c[3] = a[16]*b[3]
{
double q[3];
q[0]=(a[ 0]*b[0])+(a[ 4]*b[1])+(a[ 8]*b[2])+(a[12]);
q[1]=(a[ 1]*b[0])+(a[ 5]*b[1])+(a[ 9]*b[2])+(a[13]);
q[2]=(a[ 2]*b[0])+(a[ 6]*b[1])+(a[10]*b[2])+(a[14]);
for(int i=0;i<3;i++) c[i]=q[i];
}
void matrix_subdet (double *c,double *a); // c[16] = all subdets of a[16]
double matrix_subdet ( double *a,int r,int s); // = subdet(r,s) of a[16]
double matrix_det ( double *a); // = det of a[16]
double matrix_det ( double *a,double *b); // = det of a[16] and subdets b[16]
void matrix_inv2 (double *c,double *a); // c[16] = a[16] ^ -1
void matrix_subdet (double *c,double *a)
{
double q[16];
int i,j;
for (i=0;i<4;i++)
for (j=0;j<4;j++)
q[j+(i<<2)]=matrix_subdet(a,i,j);
for (i=0;i<16;i++) c[i]=q[i];
}
double matrix_subdet ( double *a,int r,int s)
{
double c,q[9];
int i,j,k;
k=0; // q = sub matrix
for (j=0;j<4;j++)
if (j!=s)
for (i=0;i<4;i++)
if (i!=r)
{
q[k]=a[i+(j<<2)];
k++;
}
c=0;
c+=q[0]*q[4]*q[8];
c+=q[1]*q[5]*q[6];
c+=q[2]*q[3]*q[7];
c-=q[0]*q[5]*q[7];
c-=q[1]*q[3]*q[8];
c-=q[2]*q[4]*q[6];
if (int((r+s)&1)) c=-c; // add signum
return c;
}
double matrix_det ( double *a)
{
double c=0;
c+=a[ 0]*matrix_subdet(a,0,0);
c+=a[ 4]*matrix_subdet(a,0,1);
c+=a[ 8]*matrix_subdet(a,0,2);
c+=a[12]*matrix_subdet(a,0,3);
return c;
}
double matrix_det ( double *a,double *b)
{
double c=0;
c+=a[ 0]*b[ 0];
c+=a[ 4]*b[ 1];
c+=a[ 8]*b[ 2];
c+=a[12]*b[ 3];
return c;
}
void matrix_inv(double *c,double *a)
{
double d[16],D;
matrix_subdet(d,a);
D=matrix_det(a,d);
if (D) D=1.0/D;
for (int i=0;i<16;i++) c[i]=d[i]*D;
}
//---------------------------------------------------------------------------
//--- render ----------------------------------------------------------------
//---------------------------------------------------------------------------
void TForm1::draw()
{
int i;
double xs=ClientWidth,ys=ClientHeight; // just my GL view resolution
double p[3],im[16]; // some temp point and inverse matrix
double b[4][3]; // your rectangle
matrix_inv(im,m); // im=Inverse(m)
glClear(GL_COLOR_BUFFER_BIT);
glDisable(GL_CULL_FACE);
glDisable(GL_DEPTH_TEST);
glMatrixMode(GL_PROJECTION);
glLoadIdentity();
glTranslated(-0.5,+0.5,0.0);
glScaled(2.0/xs,-2.0/ys,1.0);
glMatrixMode(GL_MODELVIEW);
glLoadIdentity();
// render transformed a and compute its BBOX into b
glColor3d(0.0,0.5,0.5); // aqua
glBegin(GL_LINE_LOOP);
for (i=0;i<4;i++)
{
// transform and render rectangle a
matrix_mul_vector(p,im,a[i]); // p = inverse(m)*a[i]
glVertex2dv(p);
// compute transformed BBOX b[0]=min(p) b[2]=max(p)
if (i==0)
{
vector_copy(b[0],p);
vector_copy(b[2],p);
}
if (b[0][0]>p[0]) b[0][0]=p[0];
if (b[2][0]<p[0]) b[2][0]=p[0];
if (b[0][1]>p[1]) b[0][1]=p[1];
if (b[2][1]<p[1]) b[2][1]=p[1];
}
glEnd();
// convert BBOX b[0],b[2] to rectangle
b[1][0]=b[2][0];
b[1][1]=b[0][1];
b[3][0]=b[0][0];
b[3][1]=b[2][1];
// render transformed b
glColor3d(0.8,0.0,0.0); // red
glBegin(GL_LINE_LOOP);
for (i=0;i<4;i++) glVertex2dv(b[i]); glEnd();
// untransform b to rectangle local coordinates
for (i=0;i<4;i++) matrix_mul_vector(b[i],m,b[i]); // b[i] = m*b[i]
// render a,b in local coordiantes (untransformed)
glColor3d(0.0,0.25,0.25); // aqua
glBegin(GL_LINE_LOOP);
for (i=0;i<4;i++) glVertex2dv(a[i]);
glEnd();
glColor3d(0.4,0.0,0.0); // red
glBegin(GL_LINE_LOOP);
for (i=0;i<4;i++) glVertex2dv(b[i]);
glEnd();
glFlush();
SwapBuffers(hdc);
}
//---------------------------------------------------------------------------
And tweaked the transformation notation of yours until it matched your preview:
the more lighter colors correspond to stuff transformed by m and the darker are without transformation (rectangle local).
I figured out that your notation for transformation of p into canvas coordinates is:
p' = Inverse(m)*p
So to transform back you would normaly be done with:
p = m*p`
However if your inverse matrix is just pseudo inverse (like this) to get it working you need to divide the inverted matrix by scale.
So if you got the points of BBOX you just multiply m with them and the result is what you seek.
