Im currently working on an SSL crawler and I'm wondering which is the best or safest way to kill the spawned threads. In fact, I fork threads based on available URLs fetched from DB. I have a class thread controller which monitors all spawned thread and decide whether to fork more based on URLs availability? I need advice regarding killing thread safely?
here is the thread controller:
public void startThreadsV4() throws InstantiationException, IllegalAccessException {
//creating a pool of maxThreads threads
//ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(maxThreads);
int m = maxThreads - nThreads;
if (availableURLs < m) {
m = availableURLs;
}
System.out.println("Starting "+ m + " threads");
for (int n = 0; n < m; n++) {
ControllableThread thread = (ControllableThread) threadClass.newInstance();
thread.setThreadController(this);
thread.setMessageReceiver(receiver);
thread.setId(nThreads++);
thread.setDbhandler(dbhandler);
thread.setIOhandler(IOhandler);
thread.setIgnoresundomains(ignoresubdomain);
thread.start();
allThreads.add(thread);
}
}
and here is the thread logic:
public void run() {
///first thing is to check if the The thread Controller requested
IOhandler io = new IOhandler();
try {
//Object newTask = dbhandler.getNCURLandMarkTransaction();
Object newTask = dbhandler.getNCURL();
System.out.println("new URL -->"+ newTask);
while (newTask != null) {
if (tc.getStop() == 1) {
return;
} else {
//treatment in case of ignore sub domains
if( ignoresundomains == 1 ){
String[] temp = new String[2];
temp = io.getHostnamefromURL(newTask.toString());
if( dbhandler.HostExist(temp[0],temp[1])){
return;
}
}
dbhandler.markURLinprogress(newTask.toString());
//notify controler of what we do
mr.receiveMessage(newTask, id);
//process the new task (URL)
System.out.println("Processing" + newTask);
process(newTask, tc.getLastrecord(), tc, dbhandler,IOhandler);
//dec availabe urls
tc.decAvailabeURLs();
dbhandler.markURLcrawled(newTask.toString());
if (tc.getMaxThreads() > tc.getRunningThreads()) {
try {
tc.startThreadsV4();
} catch (InstantiationException | IllegalAccessException ex) {
Logger.getLogger(ControllableThread.class.getName()).log(Level.SEVERE, null, ex);
}
}
//System.out.println("DONE");
//send finished message to controller
tc.finished(id);
newTask = dbhandler.getNCURL();
}
}
} catch (SQLException ex) {
Logger.getLogger(ControllableThread.class.getName()).log(Level.SEVERE, null, ex);
}
}