I'm new to express / node and trying to build an api endpoint to retrieve html from websites :

// load express
const express = require('express');

// get app & port
const app = express();
const port = process.env.PORT || 3000 ;

// get current url parameter and process
app.get('/', function(req, res) {

const url = req.query.url;
const puppeteer = require('puppeteer') ;

(async () => {

    try {

        var browser = await puppeteer.launch({
            headless: true,
            devtools: false,
            ignoreHTTPSErrors: true,
            userDataDir: 'tmp' ,
            args: [
                '--no-sandbox',
                '--single-process',
                '--no-zygote',
                '--disable-setuid-sandbox',
                '--disable-gpu'
            ]
        });


       // build a new page
        const page = await browser.newPage()

        // set options
        await page.setViewport({width: 800, height: 600})
        await page.setDefaultNavigationTimeout( 30000 ) ;

        // start time
        var start = Date.now();

        // explore url
        try {

            // goto page
            const response = await page.goto(url, { waitUntil: 'load'} );

            // get http response code
            const response_status = response == null ? 0 : response.status() ;

            // get html
            const data = await page.evaluate(() => document.querySelector('*').outerHTML);

            // send result
            res.send({
                'success' : true,
                'url': url,
                'http_code' : response_status ,
                'time' :  Date.now() - start,
                'data' : data
            });
        }
        // catch errors
        catch( err ) {

            // get page error
            const page_error = err.toString()

            // send error
            res.send({
                'success' : false,
                'url': url,
                'error' : page_error,
                'time' :  Date.now() - start
            });
        }

    } catch (err) {

        // send error
        res.send({
            'success' : false,
            'url': url,
            'error' : err.toString(),
            'time' :  Date.now() - start
        });

    } finally {

        await browser.close();
    }

})();

})


// start app
app.listen(port);

When using the endpoint at a rate of ~1 request per second, pm2 monitoring shows an increasing http latency.

First minute :

After ~10 minutes :

The memory used seem to increase too from 140M to more (500M after few minutes):

Not sure where the memory leak could come from.

CodePudding user response：

As suggested by Matthias, the "require puppeeter" should be outside the routing function. It solved the problem. Stupid mistake.