I am currently trying to create Hierarchical Clustering using Similarity Matrix on a dataset of 35K images. I am trying to build the similarity matrix by concatenating Gabor filters, zernike moments and HOG of the images. AS the process was slow, I tried using multiprocessing and assign multiple threads. i split it into 4 threads and concatenated the result received at the end of each thread. But there is an error stating "not enough space for thread data R6016 " The program continues after I press ok button. But the task is being interrupted frequently and I need to press the ok button. Can anyone tell me how to get around it?
The code is:
from hog_feature import HOGFeature
from gabor_feature3 import GaborFeatures
from zernike_moments import ZernikeMoments
import csv
import pyexcel as pe
import time
import cv2
from multiprocessing import Process, Queue
from scipy.spatial import distance as dist
class HierarchicalCluster:
def __init__(self):
self.similarity_matrix = []
self.location = []
self.gabor = GaborFeatures()
self.zernike = ZernikeMoments()
self.hog = HOGFeature()
self.filters = self.gabor.build_filters()
self.w_g = 0.33
self.w_z = 0.33
self.w_h = 0.33
# extracting info from csv file
def extract_location(self):
with open('OSU_gps_GrahamCreek.csv', 'r') as csvfile:
reader = csv.reader(csvfile, delimiter=',')
for row in reader:
self.location.append(row[3])
return self.location
def tailend_extract(self, image1, image2):
feature_g = Queue()
feature_z = Queue()
feature_h = Queue()
feature_extract = []
# gabor process
g1 = Process(target=self.gabor.feature_extract_image1, args=(image1, feature_g, self.filters))
g2 = Process(target=self.gabor.feature_extract_image1, args=(image2, feature_g, self.filters))
# zernike process
z1 = Process(target=self.zernike.zernike_dist, args=(image1, image2, feature_z))
# hog feature
h1 = Process(target=self.hog.feature_extract, args=(image1, image2, feature_h))
# start processes
g1.start()
g2.start()
z1.start()
h1.start()
image1_gf = feature_g.get()
image2_gf = feature_g.get()
dist_z = feature_z.get()
dist_h = feature_h.get()
if dist_h < 0:
dist_h = 0.0
g1.join()
g2.join()
z1.join()
h1.join()
feature_h.close()
feature_z.close()
feature_g.close()
# calculate gabor distance
dist_g = (dist.euclidean(image1_gf,image2_gf))*(10**(-13))
pairwise_feature = (self.w_g * dist_g) + (self.w_z * dist_z) + (self.w_h * dist_h)
#print pairwise_feature
return pairwise_feature
def pairwise_similarity(self, location):
for i in range(len(location)):
s = time.time()
pairwise_similarity_matrix = []
image_p = cv2.imread(location[i], flags=cv2.COLOR_BGR2GRAY)
if i >= 1:
for j in range(0,i):
pairwise_similarity_matrix.append(self.similarity_matrix[j][i])
for j in range(i, len(location)):
if i==j:
pairwise_similarity_matrix.append(0.0)
else:
image_q = cv2.imread(location[j], flags=cv2.COLOR_BGR2GRAY)
distance = self.tailend_extract(image_p,image_q)
pairwise_similarity_matrix.append(distance)
print (str(i) + ":" + str(j))
self.similarity_matrix.append(pairwise_similarity_matrix)
#print (time.time() - s)
return self.similarity_matrix
def pairwise_similarity_test(self, image1, image2):
gabor_feature = self.gabor.get_distance(img1=image1, img2=image2)
zernike_feature = self.zernike.get_distance(img1=image1, img2=image2)
hog_feature = self.hog.get_distance(img1=image1, img2=image2)
pairwise_feature = self.w_g*gabor_feature + self.w_z*zernike_feature + self.w_h*hog_feature
print pairwise_feature
def create_csv(self, similarity_matrix):
sheet = pe.Sheet(similarity_matrix)
sheet.save_as("similarity_matrix_GC.csv")
if __name__ == '__main__':
cluster = HierarchicalCluster()
locations = cluster.extract_location()
similarity = cluster.pairwise_similarity(locations)
print similarity
cluster.create_csv(similarity)
In here the zernike, gabor and HOG are just different classes that I am using to extract vectors.
The error shows as: Error message
Thank you in advance for the help
