I am querying an endpoint and need to hit it multiple times to get all of the data. I am trying to do this recursively as I have no idea how many hits at any given time I would need to make to get the full dataset.

I am really coming unstuck due to the way node is non-blocking. I understand why it needs to be as it is single threaded. However, I am struggling implementing my code in a way that will give me the desired output.

The more im reading on here, the more im coming unstuck.

I simply need my functions to hit an endpoint repeatedly until I have pulled all of the data. Then I would like to push this data to an array. Then I would like to work with the data.

-Function to query endpoint

async function getTwitch(url = "", data = {}) {

const response = await fetch(url, {
  method: 'GET', // *GET, POST, PUT, DELETE, etc.
  headers: {
    'Content-Type': 'application/json',
    "Authorization": "Bearer superSecret",
    "Client-ID": "0asdbn3245kas364"
    // 'Content-Type': 'application/x-www-form-urlencoded',
  }
  
});
  return response.json(); // parses JSON response into native JavaScript objects
}

-Callback and Global Variables

let test_BreakCase = 10;
let streams = [];

async function getLiveStreams(cursor = ''){

  //create query parameter for pagination and build into url
  let query = (cursor !== '') ? `after=${cursor}&` : '';
  let baseURL = `https://api.twitch.tv/helix/streams?${query}first=100&language=en`;

  //make fetch req for page; when promise returns iterate it and push data into array
  getTwitch(baseURL)
  .then(data => {
    let cursor = data.pagination.cursor;
    for(let i = 0; i < data.data.length; i  ){
      streams.push(data.data[i].user_login);
    }

    //if cursor not empty recursively call getTwitch()
    if(cursor !== '' && data.pagination !== null && test_BreakCase > 10){
      getLiveStreams(cursor);
    }
  });
}

-Feeble attempt to check for values in the array

const s = await getLiveStreams().then(data => {
  console.log(streams);
});

Thank you

CodePudding user response：

I'm no expert but when I start trying to store the output of recursive functions outside the function before it's finished being called, I'm usually doing something wrong. What if you pass the streams array to the recursive function as you go? This also solves the other possible issue I noticed, which is that usually both the base and recursive cases should return something, whether it's the final result or another recursive call. We keep returning a call to getLiveStreams until the final array of data is returned, which bubbles up the call stack to the initial call to the recursive function.

async function getLiveStreams(cursor = '', streams = []) {
    //create query parameter for pagination and build into url
    let query = cursor !== '' ? `after=${cursor}&` : ''
    let baseURL = `https://api.twitch.tv/helix/streams?${query}first=100&language=en`

    const data = await getTwitch(baseURL)
    cursor = data.pagination.cursor
    for (let i = 0; i < data.data.length; i  ) {
        streams.push(data.data[i].user_login)
    }

    // base case: cursor is empty
    if (cursor === '' || data.pagination === null) {
        return streams
    }

    // recursive case
    return await getLiveStreams(cursor, streams)
}

Edit: the return in the recursive case should probably be an await too, since the function returns a promise.

Edit2: I tested this slightly re-written version of your code with jsonplaceholder and it worked:

const https = require('https')
const _URL = 'https://jsonplaceholder.typicode.com/posts/'

async function getTwitch(url) {
    return new Promise((resolve, reject) => {
        let data = ''
        https.get(url, (res) => {
            res.on('data', (chunk) => {
                data  = chunk
            })
            res.on('end', () => resolve(JSON.parse(data)))
        })
    })
}

async function getLiveStreams(i = 1, streams = []) {
    let baseURL = _URL   i.toString()

    const data = await getTwitch(baseURL)

    streams.push(data)

    // base case: we've made 50 requests
    if (i === 50) {
        return streams
    }

    // recursive case
    return await getLiveStreams(i   1, streams)
}

getLiveStreams().then(results => console.log(results))