Solved, here is the solution: show the modal window only when a link containing the .txt, .pdf, .jpg, name extension is clicked.
Links without extension are also included except those ending in a slash, if you want not to include them replace the pattern with this:
/^(.(?!.*\.txt|.*\.pdf|.*\.jpg|.*\.jpeg|.*\.png))*$/
$('.link a').each(function(){
// modal login for documents, images and files without extension
var herf = $(this).attr("href");
if(herf.match(/^(.(?!.*\.txt|.*\.pdf|.*\.jpg|.*\.jpeg|.*\.png|.*\/))*$/)) {
$(this).attr('data-bs-toggle', 'modal').attr('data-bs-target', '#exampleModal');
}
});
<table id="list">
<tr><td class="link"><a href="file.avi">avi file</a> deny</td></tr>
<tr><td class="link"><a href="file.txt">txt file </a> allow</td></tr>
<tr><td class="link"><a href="file.mp3">mp3 file </a> deny</td></tr>
<tr><td class="link"><a href="file.jpg">jpg file</a> allow</td></tr>
<tr><td class="link"><a href="file">empty file</a> deny</td></tr>
<tr><td class="link"><a href="dir/">directory/</a> allow</td></tr>
</table>
<div class="modal fade" id="exampleModal" tabindex="-1" aria-labelledby="exampleModalLabel" aria-hidden="true">
<div class="modal-dialog">
<div class="modal-content">
<div class="modal-header">
<h5 class="modal-title" id="exampleModalLabel">Modal title</h5>
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
</div>
<div class="modal-body">
...
</div>
<div class="modal-footer">
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
<button type="button" class="btn btn-primary">Save changes</button>
</div>
</div>
</div>
</div>
<link href="https://getbootstrap.com/docs/5.0/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-EVSTQN3/azprG1Anm3QDgpJLIm9Nao0Yz1ztcQTwFspd3yD65VohhpuuCOmLASjC" crossorigin="anonymous">
<link href="https://getbootstrap.com/docs/5.0/assets/css/docs.css" rel="stylesheet">
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/js/bootstrap.bundle.min.js" integrity="sha384-MrcW6ZMFYlzcLA8Nl+NtUVF0sA7MsXsP1UyJoMp4YLEuNSfAP+JcXn/tWtIaxVXM" crossorigin="anonymous"></script>