Https Request Response arriving after my Express API Response-CodePudding

I am writing an API endpoint which searches Google Book's API to add book titles to the notebooks in my MongoDB Database based on their Id's. I have it working properly, but sometimes my endpoint's response array returns missing the last item. My console logs seem to indicate my endpoint response is being sent before the Https request has finished fetching all the data. I need to solve the endpoint sometimes returning an incomplete array. I think I may need to include an async await somewhere but I'm not sure where. Any help is much appreciated!

Here is my code:

//GET a user's books with notes
const getNotebooks = async (req, res) => {

...

const books = await Book.find({ userId }).sort({createdAt: -1}) //gets all notebooks as array

let results = []

for (let i = 0; i < books.length; i  ){
    console.log(i   " - "   books[i].gId)

    const item = {
        id: books[i].userId,
        gId: books[i].gId,
        title: '',
    }
    
    const request = https.request(apiUrl   books[i].gId, async (response) => {
        
        let data = '';
        response.on('data', (chunk) => {
            data = data   chunk.toString();
        });
        response.on('end', () => {
            const body = JSON.parse(data);
            item.title = body.volumeInfo.title
            results.push(item)

            if (i == books.length - 1){

                res.status(200).json(results)

            }
        });
    })
    request.on('error', (error) => {
        console.log('An error', error);
        res.status(400).json({error: error})
    });
    request.end() 
}}

CodePudding user response：

You can use async-await with Promises only, since Node's core https module does not have any build-in promise support, you will have to convert it first into the promise format, then you can use async-await with it, i am not familiar with the standard https module but i will make an example with your code

const getNotebooks = async (req, res) => {

    //...
    
    const books = await Book.find({ userId }).sort({createdAt: -1}) //gets all notebooks as array
    
    let results = []
    
    for (let i = 0; i < books.length; i  ){
        console.log(i   " - "   books[i].gId)
    
        const item = {
            id: books[i].userId,
            gId: books[i].gId,
            title: '',
        }
        
    const options = apiUrl   books[i].gId    
        
    // now you can use await to this function which return a promise 
    await makeRequest(options, res, item, results, books )  
    }
}

// Making this function into a promise outside of the scope from your getNotebooks function.
function makeRequest(options, res, item, results, books) {
    
    return new Promise((resolve, rejects) => {
    const request = https.request(options, (response) => {
            
    let data = '';
        response.on('data', (chunk) => {
            data = data   chunk.toString();
        });
        response.on('end', () => {
            const body = JSON.parse(data);
            resolve(body);
            item.title = body.volumeInfo.title
            results.push(item)
            if (i == books.length - 1){
                res.status(200).json(results) 
            }
        });
    });
        request.on('error', (error) => {
            console.log('An error', error);
            rejects(error);
            res.status(400).json({error: error})
    });
        request.end() 
  });
};

Alternatively you can also use a more modern way to request data, for example with fetch which is now available in node, or even Axios library, if you don't want any dependency, fetch should be the way to go, an example if there is no mistake and assuming is a GET method request:

const getNotebooks = async (req, res) => {
   //...
   try {
        const books = await Book.find({ userId }).sort({createdAt: -1}) //gets all notebooks as array

        let results = []

        for (let i = 0; i < books.length; i  ) {
            console.log(i   " - "   books[i].gId)

            const item = {
                id: books[i].userId,
                gId: books[i].gId,
                title: '',
            }
        // using fetch on node js is now possible without any package to install
            const response = await fetch(apiUrl   books[i].gId)
            const data = await response.json();

            item.title = data.volumeInfo.title
            results.push(item)
            if (i == books.length - 1){
                res.status(200).json(results)    
            }
        }
    } catch (error) {
        res.status(400).json({error: error})
    };
};

As you can see you would write 2 line code instead of 20 line.