I had the same weird error but in Java. In my case, It seemed that estimateRigidTransform couldn't recognize that the two Mat images I was giving where actually 2D Point Sets. So I applied a workaround in order to convert my match points from MatOfKeyPoint to MatOfPoint2f type.
Here is the complete Java code (It's not Python, but maybe it will help you):
UPDATE: Filtering your matches is important, cause if you don't you may get an empty array as a result of the transform.
FeatureDetector detector = FeatureDetector.create(FeatureDetector.ORB);
DescriptorExtractor descriptor = DescriptorExtractor.create(DescriptorExtractor.ORB);
DescriptorMatcher matcher = DescriptorMatcher.create(DescriptorMatcher.BRUTEFORCE_HAMMING);
// Load First Image
Mat img1 = Imgcodecs.imread("img1_path", Imgcodecs.IMREAD_GRAYSCALE);
Mat img1_descriptors = new Mat();
MatOfKeyPoint img1_keypoints_mat = new MatOfKeyPoint();
// Detect KeyPoints
detector.detect(img1, img1_keypoints_mat);
descriptor.compute(img1, img1_keypoints_mat, img1_descriptors);
// Load Second Image
Mat img2 = Imgcodecs.imread("img2_path", Imgcodecs.IMREAD_GRAYSCALE);
Mat img2_descriptors = new Mat();
MatOfKeyPoint img2_keypoints_mat = new MatOfKeyPoint();
// Detect KeyPoints
detector.detect(img2, img2_keypoints_mat);
descriptor.compute(img2, img2_keypoints_mat, img2_descriptors);
// Match KeyPoints
MatOfDMatch matOfDMatch = new MatOfDMatch();
matcher.match(img1_descriptors, img2_descriptors, matOfDMatch);
// Filtering the matches
List<DMatch> dMatchList = matOfDMatch.toList();
Double max_dist = 0.0;
Double min_dist = 100.0;
for(int i = 0; i < img1_descriptors.rows(); i++){
Double dist = (double) dMatchList.get(i).distance;
if(dist < min_dist) min_dist = dist;
if(dist > max_dist) max_dist = dist;
}
LinkedList<DMatch> good_matches = new LinkedList<>();
for(int i = 0; i < img1_descriptors.rows(); i++){
if(dMatchList.get(i).distance < 3*min_dist){
good_matches.addLast(dMatchList.get(i));
}
}
// Converting to MatOfPoint2f format
LinkedList<Point> img1_points_list = new LinkedList<>();
LinkedList<Point> img2_points_list = new LinkedList<>();
List<KeyPoint> img1_keyPoints_list = img1_keypoints_mat.toList();
List<KeyPoint> img2_keyPoints_list = img2_keypoints_mat.toList();
int limit = good_matches.size();
for(int i = 0; i < limit; i++){
img1_points_list.addLast(img1_keyPoints_list.get(good_matches.get(i).queryIdx).pt);
img2_points_list.addLast(img2_keyPoints_list.get(good_matches.get(i).trainIdx).pt);
}
MatOfPoint2f img1_point2f_mat = new MatOfPoint2f();
img1_point2f_mat.fromList(img1_points_list);
MatOfPoint2f img2_point2f_mat = new MatOfPoint2f();
img2_point2f_mat.fromList(img2_points_list);
// Draw match points
Mat output = new Mat();
Features2d.drawMatches(img1, img1_keypoints_mat, img2, img2_keypoints_mat, matOfDMatch, output);
Imgcodecs.imwrite("output.png", output);
Mat result = Video.estimateRigidTransform(img1_point2f_mat, img2_point2f_mat, true);
printMat(result); // Printing the optimal affine transformation 2x3 array
// The following variables correspond to the estimateRigidTransform result as shown here: https://stackoverflow.com/a/29511091/5165833
double a = result.get(0,0)[0];
double b = result.get(0,1)[0];
double d = result.get(1,1)[0];
double c = result.get(1,0)[0];
// Solving for scale as shown in the link above
double scale_x = Math.signum(a) * Math.sqrt( (a*a) + (b*b) );
double scale_y = Math.signum(d) * Math.sqrt( (c*c) + (d*d) );
System.out.println("a = "+a);
System.out.println("b = "+b);
System.out.println("scale_x = "+scale_x);
System.out.println("scale_y = "+scale_y);
}
public static void printMat(Mat m)
{
for (int x=0; x < m.height(); x++)
{
for (int y=0; y < m.width(); y++)
{
System.out.printf("%f",m.get(x,y)[0]);
System.out.printf("%s"," ");
}
System.out.printf("\n");
}
}
