1

I have some code that I think is not working properly now because I added something to get the MIME Type (the real MIME type) using some JS. It calls checkDicomMime(file), and it asynchronously reads the files to be uploaded, and determines if the MIME type matches what I am looking for.

I think the MIME type detection is working, but because it takes time to read the files, I think the rest of the code executes before it is done reading the MIME types.

Previously, I was just checking for a file extension, and that was done synchronously so that the variables in the "reader.onload = function (evt) {" block in the function were set inline. Now, it calls the function, and the function correctly detects the MIME type, but it looks like the calling function completes and the rest of the code executes before the MIME TYPE detection completes, so it POSTS the form for each file in the list before the MIME TYPE detection is done. The total = counts.process is now zero rather than the total number of files to process, so counts and files.process and badfiles are either not getting altered, or they change only after all of the files have been posted. I checked with some debugging and it looks like they are set after the files are sent. Also, that other SO post talks about reading in just the requisite number of bytes to detect the MIME type rather than reading the whole file. Not sure exactly how to do that.

I got the DICOM checking function here: Check Dicom

And there are some discussions about MIME type detection in general using JS here:

How to check file MIME type with javascript before upload?

Relevant code is:

var counts;

// Detects when a Folder is selected, Folder, not a file.

picker.addEventListener('change', e => {

    counts = {process:0,omit:0};
    requestcounter = 0;
    responsecounter = 0;
    total = 0;
    skipotherrequests = 0;
    parsedepoch = new Date().toISOString().match(/(\d{4}\-\d{2}\-\d{2})T(\d{2}:\d{2}:\d{2})/);
    datetimestamp = parsedepoch[1] + "-" + parsedepoch[2].replace(/:/g, "-");
    //alert(datetimestamp);
    picker.setAttribute('data-timestamp', datetimestamp);

    // preprocess checking

    var badfiles = [];

    var filelist = Array.from(picker.files);


    filelist.forEach(function(file, index) {
        // add it to the list, otherwise skip it
        checkDicomMime(file);  // calls the check for MIME type.
    });

    filelist.sort(function(a,b) {
    return a.name > b.name;
    });

    total = counts.process;  // omitting the ones that do not pass verification.

    badlist = "";

    badfiles.forEach( element => badlist += '<div>' + element + '</div>' );

    for (var i = 0; i < filelist.length; i++) {

    var file = filelist[i];
    if (file.process == 0) {
        let lineitem = statusitem(file, "Skipping file:  " + file.name);
        listing.insertAdjacentHTML('beforeend', lineitem);
    }
    else {
    sendFile(file);  // sends form and file
    }
    }
});

function checkDicomMime(file) {

        var reader = new FileReader();
        reader.readAsArrayBuffer(file);

        //Fired after sucessful file read, Please read documenation for FileReader
        reader.onload = function (evt) {
            if (evt.target.readyState === FileReader.DONE) {

                var array = new Uint8Array(evt.target.result);
                var s = "";
                var start = 128, end = 132;
                for (var i = start; i < end; ++i) {
                    s += String.fromCharCode(array[i]);
                }

                if (s == "DICM") {

                    alert("DICM a valid dicom file");

                    file.process = 1;
                    counts.process++;
                }

                else {

                    alert("DICM not found");
                     file.process = 0;
                     counts.omit++;
                     badfiles.push (file.name);
                }
            }
        }
}

The beginning of then sendFile function is:

sendFile = function(file) {

    if (skipotherrequests == 0) {

    var timestamp  = picker.dataset.timestamp;
    var formData = new FormData();
    // Set post variables 

    requestcounter = requestcounter + 1;
    formData.set('timestamp', timestamp); // One object file
    formData.set('counter', requestcounter);
    formData.set('total', total); 
    formData.set('type', type); 
    formData.set('webkitpath', file.webkitRelativePath); // One object file
    formData.set('file', file); // One object file
    //console.log(file);

    var request = new XMLHttpRequest();

    request.responseType = 'json';

    // HTTP onload handler

    request.onload = function() {

        if (request.readyState === request.DONE) {
SScotti
  • 2,158
  • 4
  • 23
  • 41

2 Answers2

1

Now, it calls the function, and the function correctly detects the MIME type, but it looks like the calling function completes and the rest of the code executes before the MIME TYPE detection completes, so it POSTS the form for each file in the list before the MIME TYPE detection is done.

You can change checkDicomMime to promise and wait until all files are checked.

Then you can continue processing them in a loop and send valid ones as you did.

Of course this requires a bit of code refactoring.

Example

const picker = document.querySelector("#file");
const listing = document.querySelector("#listing");
const button = document.querySelector("#button");

picker.addEventListener('change', async event => {

  const counts = {
    process: 0,
    omit: 0
  };
  let requestcounter = 0;
  let responsecounter = 0;
  let total = 0;
  let skipotherrequests = 0;
  const [, datePart, timePart] = new Date().toISOString().match(/(\d{4}\-\d{2}\-\d{2})T(\d{2}:\d{2}:\d{2})/);
  const datetimestamp = `${datePart}-${timePart.replace(/:/g, "-")}`;
  picker.setAttribute('data-timestamp', datetimestamp);

  const files = Array.from(event.detail || event.target.files);
  const processList = await Promise.all(files.map(file => checkDicomMime(file)));

  processList.sort((prev, next) => {
    return prev.fileName > next.fileName;
  });

  const badlist = processList.filter(({
      isBadFile
    }) => isBadFile)
    .reduce((acc, result) => acc += `<div>${result.fileName}</div>`, '');

  const timestamp = picker.dataset.timestamp;
  for (let result of processList) {
    const file = result.file;
    const type = file.type;
  
    if (result.isBadFile) {
      let lineitem = statusitem(file, `Skipping file: ${result.fileName}`);
      listing.insertAdjacentHTML('beforeend', lineitem);
      continue;
    }
  
    console.log('sending file', file)
    requestcounter = requestcounter + 1;
    await sendFile(file, timestamp, requestcounter, total, type);
  }

});

function statusitem(file, text) {
  return `<div>${text}</div>`;
}

function checkDicomMime(file) {
  const fileReader = new FileReader();
  return new Promise((resolve, reject) => {

    fileReader.readAsArrayBuffer(file);
    fileReader.onload = function(event) {

      const target = event.target;
      const array = new Uint8Array(target.result);
      const start = 128
      const end = 132;
      const str = [...array.slice(128, 132)].map(value => String.fromCharCode(value)).join('');

      const result = {
        file,
        fileName: file.name,
        isBadFile: true
      }

      if (str == "DICM") {
        result.isBadFile = false;
      }

      fileReader.onload = null;
      resolve(result);
    }
  })
}

const sendFile = function(file, timestamp, requestcounter, total, type) {
  return new Promise((resolve, reject) => {
    const formData = new FormData();
    formData.set('timestamp', timestamp);
    formData.set('counter', requestcounter);
    formData.set('total', total);
    formData.set('type', type);
    formData.set('webkitpath', file.webkitRelativePath);
    formData.set('file', file);

    const request = new XMLHttpRequest();
    request.responseType = 'json';
    request.onload = function() {

      if (request.readyState === request.DONE) {
        resolve();
      }
    }
  })
}

function createInvalidFile() {
  const data = [new Uint8Array(Array(132).fill(0))]
  const file = new File(data, 'invalid-file.txt',{
  type: "text/plain"
  });
  return file;
}

function createValidFile() {
  const data = [new Uint8Array(Array(128).fill(0)), new Uint8Array([68, 73, 67, 77])]
  const file = new File(data, 'valid-file.txt', {
  type: "text/plain"
  });
  return file;
}

button.addEventListener("click", event => {
  const customEvent = new CustomEvent('change', {
    detail: [createInvalidFile(), createValidFile()]
  });
  picker.dispatchEvent(customEvent);
})
<input id="file" type="file" multiple>
<div id="listing"></div>
<button id="button">Send test files</button>
Józef Podlecki
  • 10,453
  • 5
  • 24
  • 50
  • 1
    Thanks. I incorporated some of that with modifications and testing with an actual folder containing .dcm files and other files. One reason for checking client side is to avoid uploading non-dicom files since we don't need those, and they might be quite large or contain .exe files that we don't want. Some dicom files do not have a .dcm extension, and that also. pertains to other MIME types as well. PHP can detect the actual MIME type server side and we are doing that also. If I want to include the working code now, should I post in my question, or add an answer, even though accepted yours? – SScotti May 20 '20 at 18:06
  • Also, I did not have to use promises for the actual posting of the files via AJAX, just the detection of MIME types client side. The server can handle having the files fired off asynchronously in rapid succession. That required some fairly complex code since it uses DCMTK or parsing the headers and sending them off to a PACS. – SScotti May 20 '20 at 18:08
  • I think you should add separate answer which tells how you solved it. – Józef Podlecki May 20 '20 at 20:33
  • Thanks. I had to also add something to just read the required portion of the files on the client side. var blob = file.slice(0, 132); //read the first 4 bytes of the file fileReader.readAsArrayBuffer(blob); It is pretty complicated, but if I just included the framwork, might not be too much. It is pretty fast really because the server processes them pretty quickly. An alternative would be to just upload a .zip file and unzip it on the server side, but this is for radiology studies that you would get burned onto a CD by a clinic or a hospital. This allows you to upload the CD. – SScotti May 21 '20 at 01:50
  • Posted much of my code, except for the server side, but showed expect response. If you have a dicom file (available here), you can test: https://medistim.com/dicom/ – SScotti May 21 '20 at 02:54
1

Already accepted an answer, but posting modified code, which seems to work well now. If you actually have a folder with some .dcm files (with or without the file extension), it should exclude anything that is not a real .dcm file, and that could be extended for other types of files as explained in the other post that I referenced.

There also is a library that will do this for you, although not sure that it just reads in the first few bytes needed to detect the MIME type:

GitHub Library to Detect MIME Client Side

Also, if you run the snippet, it will fire off a bunch of AJAX requests with the FORM data set, e.g.:

-----------------------------391231719611056787701959262038
Content-Disposition: form-data; name="method"
UploadFolder
-----------------------------391231719611056787701959262038
Content-Disposition: form-data; name="timestamp"
2020-05-21-02-21-45
-----------------------------391231719611056787701959262038
Content-Disposition: form-data; name="counter"
3
-----------------------------391231719611056787701959262038
Content-Disposition: form-data; name="total"
14
-----------------------------391231719611056787701959262038
Content-Disposition: form-data; name="anon_normal"
<?php echo $_GET['anon_normal'] ?>
-----------------------------391231719611056787701959262038
Content-Disposition: form-data; name="userid"
<?php echo $_GET['userid'] ?>
-----------------------------391231719611056787701959262038
Content-Disposition: form-data; name="type"
-----------------------------391231719611056787701959262038
Content-Disposition: form-data; name="webkitpath"
dicomtest/28896580
-----------------------------391231719611056787701959262038
Content-Disposition: form-data; name="file"; filename="dicomtest/28896580"

. . . .

The progress counter and other features won't work here because there is no response from the server. The PHP script would normally return JSON:

file    Object { name: "28896579.dcm", size: 547440, type: "application/dicom", … }
name    "28896579.dcm"
size    547440
type    "application/dicom"
ext "dcm"
status  "Uploaded"
counter "2"

until the "last" file is processed, although that isn't always the last response, and that returns something like:

file    Object { name: "28896590.dcm", size: 547436, type: "application/dicom", … }
name    "28896590.dcm"
size    547436
type    "application/dicom"
ext "dcm"
status  "Done"
results "bunch of HTML"

You really need some .dcm files, with or without the extension to test because it is basically going to reject any non-dicom files.

// Global variables

let picker = document.getElementById('picker');
let listing = document.getElementById('listing');
let progress_text = document.getElementById('progress_text');
let preprocess_notice = document.getElementById('preprocess_notice');
let results = document.getElementById('uploadresults');
let box = document.getElementById('box');
let elem = document.getElementById("myBar");
let loader = document.getElementById("loader");
let userid = document.getElementById("userid").value;
var anon_normal = document.getElementById("anon_normal").value;
var requestcounter;
var responsecounter;
var total;
// var excludedextensions = [".exe",".zip",".pdf",".jpg",".jpeg",".png",".gif",".doc",".docx", ".xml"];
var parsedepoch;
var datetimestamp;
var skipotherrequests;
var counts;

 
function checkDicomMime(file) {

  const fileReader = new FileReader();
  return new Promise((resolve, reject) => {
 var blob = file.slice(0, 132); //read enough bytes to get the DCM header info
 fileReader.readAsArrayBuffer(blob);
    //fileReader.readAsArrayBuffer(file);
    fileReader.onload = function(event) {

      const target = event.target;
      const array = new Uint8Array(target.result);
      const start = 128
      const end = 132;
      const str = [...array.slice(128, 132)].map(value => String.fromCharCode(value)).join('');

      const result = {
        file,
        fileName: file.name,
        isBadFile: true
      }

      if (str == "DICM") {
        result.isBadFile = false;
        counts.process++;
      }
      else {
       counts.omit++;
      }

      fileReader.onload = null;
      resolve(result);
    }
  });
}


picker.addEventListener('change', async event => {

 results.innerHTML = "";
 // Reset previous upload progress
    elem.style.width = "0px";
    listing.innerHTML = "";
    // Display image animation
    loader.style.display = "block";
    loader.style.visibility = "visible";
    preprocess_notice.innerHTML = "";
    //
 counts = {
  process:0,
  omit:0
 };
 requestcounter = 0;
 responsecounter = 0;
 total = 0;
 skipotherrequests = 0;
 const parsedepoch = new Date().toISOString().match(/(\d{4}\-\d{2}\-\d{2})T(\d{2}:\d{2}:\d{2})/);
 const datetimestamp = parsedepoch[1] + "-" + parsedepoch[2].replace(/:/g, "-");
 //alert(datetimestamp);
 picker.setAttribute('data-timestamp', datetimestamp);
 
 // Reset previous upload progress
    elem.style.width = "0px";
    listing.innerHTML = "";
    // Display image animation
    
    loader.style.display = "block";
    loader.style.visibility = "visible";
    
 let files = Array.from(picker.files);
 
    const processList = await Promise.all(files.map(file => checkDicomMime(file)));
    
 processList.sort((prev, next) => {
     return prev.fileName > next.fileName;
   });
   
   const badlist = processList.filter(({
      isBadFile
    }) => isBadFile)
    .reduce((acc, result) => acc += '<li>' +result.fileName + '</li>', '')

   total = counts.process; 
   if (counts.omit > 0) preprocess_notice.innerHTML = '<div style = "color:red;">Omitting ' + counts.omit + ' file(s) that did not pass criteria"</div><ol>' + badlist + '</ol>';
   
    for (let result of processList) {
  const file = result.file;
  const type = file.type;
  //console.log(result);
  if (!result.isBadFile) {
  //console.log('sending file', file)
  sendFile(file, datetimestamp, total, type);
  }
 }

});

statusitem = function(file, status) {

 let html = '<li><span>' + file.name + '</span><span>' + file.size + ' bytes</span><span>' + file.type + '</span><span>' + status + '</span></li>';
 return html;
}

// Function to send a file, call PHP backend

var sendFile = function(file, timestamp, total, type) {

 if (skipotherrequests == 0) {
 //console.log(file);
    const formData = new FormData();
    // Set post variables 
    requestcounter = requestcounter + 1;
    formData.set('method', "UploadFolder"); // One object file
    formData.set('timestamp', timestamp); // One object file
    formData.set('counter', requestcounter);
    formData.set('total', total); 
    formData.set('anon_normal', anon_normal); 
    formData.set('userid', userid); 
    formData.set('type', type); 
    formData.set('webkitpath', file.webkitRelativePath); // One object file
    formData.set('file', file); // One object file
 //console.log(file);
  
    const request = new XMLHttpRequest();

    request.responseType = 'json';

    // HTTP onload handler
    
    request.onload = function() {
     
        if (request.readyState === request.DONE) {
        
            if (request.status === 200) {
            
             progress_text.innerHTML = file.name + " (" + (responsecounter + 1) + " of " + total + " ) ";
                //console.log(request.response);
    if (request.response.status != "Uploaded" || request.response.status != "Done" ) {
    skipotherrequests = 1;
    }
                // Add file name to list
                
                item = statusitem(request.response.file, request.response.file.status);
                listing.insertAdjacentHTML('beforeend', item);
                
    responsecounter++;
                // progress_text.innerHTML = request.response.file.name + " (" + responsecounter + " of " + total + " ) ";

                // Show percentage
                box.innerHTML = Math.min(responsecounter / total * 100, 100).toFixed(2) + "%";

                // Show progress bar
                elem.innerHTML = Math.round(responsecounter / total * 100, 100) + "%";
                elem.style.width = Math.round(responsecounter / total * 100) + "%";
                
                if (responsecounter >= total) {
                progress_text.innerHTML = "Sending " + total + " file(s) is done!";
                loader.style.display = "none";
                loader.style.visibility = "hidden";
             }
              if (request.response.file.status == "Done") {
                 results.innerHTML = request.response.results;
                }

            }
            else {
             skipotherrequests = 1;
             //alert("error with AJAX requests");
            }
        }
    }

    // Do request, Sent off to the PHP Controller for processing
    
    request.open("POST", 'OrthancDevController.php');
    request.send(formData);
    }
    else {
     // already aborted, probably never gets here because all of the requests are probably sent before skipotherrequests gets set to 1.
    }
}
code {
    font-family: Roboto Mono, monospace;
    font-size: 90%;
}

.picker {
    background-color: #eee;
    padding: 1em;
}


#box {
    color: #005aa0;
    font-size: 2rem;
    font-weight: bold;
    font-size:20px;
}

#myProgress {
    width: 100%;
    height: 30px;
    background-color: #ddd;
    border-radius: 5px;
}

#myBar {
    width: 1%;
    height: 30px;
    /* background-color: #4CAF50; */
    background-color: #e24718;
    text-align: center;
    vertical-align: middle;
    font-weight: bold;
    border-radius: 5px;
}

#loader {
    display: none;
    visibility: hidden;
}
#preprocess_notice {
text-align: left;
width: max-content;
margin: auto auto;

}

.dz-message  {
border-style:dotted;
padding:30px;
}
#ZipUpload {
background:white;

}
#dicomuploader {
background:white;
text-align:center;

}
#uploadinstructions {
text-align: left;
margin: 0 10px 0 10px;
}
#listing {

height: 100px;
overflow: scroll;
margin: auto;
padding: 10px 20px 10px 20px;
list-style-position: inside;

}
#listing li span, #statusheader span {

 display:inline-block;
 overflow:hidden;
 text-overflow: ellipsis;
 border:1px solid black;
 border-collapse:collapse;
 height: 20px;
 white-space: nowrap;
 padding: 0 5px 0 5px;
}
#listing li span:first-child, #statusheader span:first-child {

 width:150px;
 text-align:left;
}
#listing li span:nth-child(2), #statusheader span:nth-child(2) {
 width:100px;
 text-align:right;

}
#listing li span:nth-child(3), #statusheader span:nth-child(3) {
 width:150px;
 text-align:left;
}
#listing li span:nth-child(4), #statusheader span:nth-child(4) {
 width:200px;
 text-align:left;
}
#statusheader {
background:black;
color:white;
width: max-content;
margin: auto;
}
#statusheader span {
 border:1px solid white;
}
<div class="loadcontent" id = "dicomuploader">
 <h2>
  Upload Study To Server
 </h2>
 <p>
 In order to upload a study, please check the following:
 <ol id ="uploadinstructions">
 <li>You have a complete study (unpacked / unzipped ) in a folder on a CD or on your computer.</li>
 <li>Typically, there will be several folders with files there that end in .dcm, although they may not have a file extension.</li>
 <li>Using the button below, select the folder containing the files you need to upload, and then the files will upload.  If there is an error, a message will be displayed.  It typically takes a minute or two for the study to be available on the server.</li>
 <li>The entire folder should upload, including any contained subfolders.</li>
 </ol>
 </p>

 <h3>
  Choose Folder
 </h3>
 <div class="picker">
  <input type="file" id="picker" name="fileList" webkitdirectory multiple data-timestamp = "">
 </div>
 <!-- for the anon vs. normal upload, also userid and token, passed in -->
 
 <input type="hidden" id="anon_normal" name="anon_normal" value = "<?php echo $_GET['anon_normal'] ?>" >
 <input type="hidden" id="userid" name="userid" value = "<?php echo $_GET['userid'] ?>" >
 <input type="hidden" id="upload_auth_token" name="upload_auth_token" value = "<?php echo $_GET['upload_auth_token'] ?>" >

 <div>
  Percentage Processed
 </div>
 <span id="box">0%</span> 
 <div style="color:red;font-size:14px;">(there will be a pause before 100% while storing the study), please wait.</div>
 <h5>
  Percentage Uploaded
 </h5>
 <div id="myProgress">
  <div id="myBar"></div>
 </div>
 <h5>
  Sent File . . <span id = "progress_text"></span>
 </h5>
 <h3>
  Files Uploaded
 </h3>
<div id="preprocess_notice"></div> 
<div id = "statusheader"><span>File Name</span><span>File Size</span><span>MIME Type</span><span>Status</span></div>
 <ol id="listing"></ol> 
 <div id="uploadresults"></div>

<img id="loader" src="loader.gif">
</div> 
SScotti
  • 2,158
  • 4
  • 23
  • 41