I've ended using a different method then the suggestions.
I'm using matrix determinants where: (T1, T2, T3) are points that form a triangle and X is the point i want to know if it's inside this triangle, then i simple calculate 3 determinants where:
d1 = det([T1 T2 X])
d2 = det([T1 X T3])
d3 = det([T1 T2 X])
If all determinants are the same sign, then the point is inside the triangle.
Now i form a list of triangles based on the selection area and check if the point is inside one of those triangles.
this.Detector.triangleDetector = function(position, triangleArray){
for(var idxString in triangleArray){
var index = parseInt(idxString);
if(this.pointInTriangle(position, triangleArray[index].coords1, triangleArray[index].coords2, triangleArray[index].coords3))
return true;
}
return false;
}
The function pointInTriangle(x,t1,t2,t3)
does the determinant verification.
this.Detector.pointInTriangle = function(x,T1,T2,T3){
var array1 = [coord1.x ,coord1.y ,coord1.z];
var array2 = [coord2.x ,coord2.y ,coord2.z];
var array3 = [coord3.x ,coord3.y ,coord3.z];
var zero = 0;
var A = [[zero,zero,zero],[zero,zero,zero],[zero,zero,zero]];
var d1,d2,d3;
A[0][0] = position.x;
A[0][1] = position.y;
A[0][2] = position.z;
A[1][0] = array2[0];
A[1][1] = array2[1];
A[1][2] = array2[2];
A[2][0] = array3[0];
A[2][1] = array3[1];
A[2][2] = array3[2];
d1 = MyMath.determinant(A,3);
A[0][0] = array1[0];
A[0][1] = array1[1];
A[0][2] = array1[2];
A[1][0] = position.x;
A[1][1] = position.y;
A[1][2] = position.z;
d2 = MyMath.determinant(A,3);
A[1][0] = array2[0];
A[1][1] = array2[1];
A[1][2] = array2[2];
A[2][0] = position.x;
A[2][1] = position.y;
A[2][2] = position.z;
d3 = MyMath.determinant(A,3);
if((d1>=0 && d2 >=0 && d3>=0) || (d1<=0 && d2 <=0 && d3<=0)){
return true;
}
return false;
};