To check how many checkboxes are checked, you can simply use:
var checked = $('#mytable').find(':checked').length;
This counts how many checked elements there are within your #mytable
element. If this returns 0
then we know that none are checked, so we can display the alert:
if (!checked)
alert('...');
Demo
$('button').on('click', function() {
var checked = $('#mytable').find(':checked').length;
if (!checked)
alert('No checkboxes are checked!');
else
alert(checked + ' checkboxes are checked!');
});
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.1/jquery.min.js"></script>
<table id="mytable">
<tr><td><input type="checkbox" name="one" value="1" /></td></tr>
<tr><td><input type="checkbox" name="two" value="2" /></td></tr>
<tr><td><input type="checkbox" name="three" value="3" /></td></tr>
</table>
<button type="button">Check</button>