array from recursive fetch request not returning properly

Question

I've trying to learn cursor-based pagination with Slack's API. My goal is to return an array of all messages using the channel.history method.

I have a javascript recursive fetch function, but I can't seem to get the local variable "final" to return properly.

The part where it logs "Pagination results successful" is logging and returning the array as expected with a length of 204.

When "final" is called upon outside of that scope, the length is 0.

I've tried experimenting with where I return the variable final, but can't seem to get it working. I think it has to do with not using a callback function, but I wasn't sure where to implement that.

Here's what I have (with my Slack token removed).

function paginate() {
  let final = [];

  let selectChannel = function(ts) {
    fetch('https://slack.com/api/channels.history?token=MY_TOKEN&channel=C6W9FH2S0&latest=' + ts)
      .then(response => response.json())
      .then(responseData => {
        let messages = responseData.messages;
        if (!responseData.has_more) {
          final.push.apply(final, messages);
          console.log('Pagination results successfull, 204 items in the array!', final);
          return final;
        } else {
          final.push.apply(final, messages);
          selectChannel(messages[messages.length - 1].ts);
        }
        return final;
      })
      .catch(error => {
        console.log('Error fetching and parsing data', error);
      });
  }

  selectChannel(new Date());
  // Returning as 0, when I am expecting 204
  console.log("Final Output", final.length);
  return final;
}


var x = paginate();
// By extention, this is returning as 0, when I am expecting 204 as well
console.log("Output", x.length);

Fair enough, its an experiment project so I didn't put time into variable naming. Output is probly a better idea. I thought the .then was addressing the async aspect. Also ty for the quick replies. — DJBrandoK, Sep 24 '17 at 18:38
Please have a look at this https://stackoverflow.com/questions/14220321/how-do-i-return-the-response-from-an-asynchronous-call — palaѕн, Sep 24 '17 at 18:39

Joel Harkes · Accepted Answer · 2017-09-24T19:23:35.113

It is because code inside the then function is executed later (hello async await :).

fetch() returns a promise of executing some point in the future and to can return this promise and add another then() method and in there console.log()

Here is your code blocks in order of execution, this might clarify things:

function paginate() {  /* 2 */
  let final = [];

  let selectChannel = function(ts) { /* 4 */
    fetch('https://slack.com/api/channels.history?token=MY_TOKEN&channel=C6W9FH2S0&latest=' + ts)
      .then(response => response.json() /* 8 */)
      .then(responseData => { /* 9 */
        let messages = responseData.messages;
        if (!responseData.has_more) {
          final.push.apply(final, messages);
          console.log('Pagination results successfull, 204 items in the array!', final);
          return final;
        } else {
          final.push.apply(final, messages);
          selectChannel(messages[messages.length - 1].ts);
        }
        return final;
      })
      .catch(error => { /* 10 */
        console.log('Error fetching and parsing data', error);
      });
  }

  selectChannel(new Date()); /* 3 */
  // Returning as 0, when I am expecting 204
  console.log("Final Output", final.length); /* 5 */
  return final; /* 6 */
}


var x = paginate(); /* 1 */
// By extention, this is returning as 0, when I am expecting 204 as well
console.log("Output", x.length); /* 7 */

As you can see step 7 logs the x.length result while, only at step 9 this result is finaly filled (the final variable is filled in step/code-block 9)

In an async world your code would (have to) look like this:

async function selectChannel() {
    try {
        var response = await fetch('https://slack.com/api/channels.history?token=MY_TOKEN&channel=C6W9FH2S0&latest=' + ts);
        var responseData = await response.json();
        let messages = responseData.messages;
        if (!responseData.has_more) {
            final.push.apply(final, messages);
            console.log('Pagination results successfull, 204 items in the array!', final);
            return final;
        } else {
            final.push.apply(final, messages);
            await selectChannel(messages[messages.length - 1].ts);
        }
        return final;
    } catch (error) {
        console.log('Error fetching and parsing data', error);
    }
}

async function execute() {
    var finalData = await selectChannel(new Date());
    console.log("Final Output", finalData.length);
}

execute() // optionally: .then(x=> console.log('we are finished executing!'));

thank you for the detailed reply and the explanation (still re-reading so I can understand. I had to add a "ts" argument to selectChannel and have the execute function return finalData to get it to return someone outside of selectChannel. It now returns 100 when I am hoping for it to return 204. You've answered my question, but now I need to find out why its only accepting the data for the first fetch call and not acting recursively. Thank you very much!! — DJBrandoK, Sep 24 '17 at 20:20
I tweaked the code you provided and got my goal using this https://jsfiddle.net/gp6f8Lyv/ — DJBrandoK, Sep 24 '17 at 20:31

array from recursive fetch request not returning properly

1 Answers1