I am having trouble deploying my web scraping function and do not know how to fix the issue.

Index.js

const functions = require("firebase-functions");
const pup = require("puppeteer"); 


const WebsiteData = require('./schema');  
  

exports.scrape = functions
    .runWith({ memory: '1GB' })
    .pubsub.schedule('0 0 * * *')
    .onRun(async (context) => {
        const browser = await pup.launch({
            headless: true,
            args: ['--no-sandbox', '--disable-setuid-sandbox']
        });

        const page = await browser.newPage();

        await page.setUserAgent(
        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.125 Safari/537.36'
        )

        var pages = [
            {
                name: 'aave-protocol',
                fee:'0.09',
                url: 'https://aave.com',
            },
            {
                name: 'uniswap-v2',
                fee:'0.30',
                url: 'https://uniswap.org',
            },
            {
                name: 'dydx',
                fee:'0.00',
                url: 'https://dydx.exchange',
            },
            {
                name: 'dodo-bsc',
                fee:'0.00',
                url: 'https://dodoex.io',
            },
            {
                name: 'cream-finance',
                fee:'0.03',
                url: 'https://cream.finance',
            },
            {
                name: 'multiplier-finance',
                fee:'0.06',
                url: 'https://multiplier.finance',
            }
        
        ]
        var result = [];

        for (var each in pages) {
            await page.goto(`https://www.dapp.com/app/${pages[each].name}`, { waitUntil: 'networkidle0' })
        

            var users = await page.evaluate(() => {
                return document.querySelector('#app > div.app-container > div.root.dapp-detail > section.detail-page-outer > div > div.left-sec > div.stats-card > div.chart-section > div:nth-child(2) > div:nth-child(4) > span.value > span').innerText
            })
            var volume = await page.evaluate(() => {
                return document.querySelector('#app > div.app-container > div.root.dapp-detail > section.detail-page-outer > div > div.left-sec > div.stats-card > div.chart-section > div:nth-child(3) > div:nth-child(4) > span.value > span:nth-child(2)').innerText
        })

        var obj = { "name": `${pages[each].name}`, "nDUniqueUsers": `${users}`, "nDTransactions": `${volume}`, "fee":  `${pages[each].fee}`, "url": `${pages[each].url}`};
            result.push(obj);
        }


        
        await browser.close();

        

          const websiteMongo = new WebsiteData({ sites: result });
        
        await websiteMongo.save(function(err,data){
            if (err) {
                console.log(err);
                return null;
            }
        });    
        
        console.log("Done.")
        return null;

    });

The function is meant to use puppeteer to open up around 5 pages collect the info and upload the data to a MongoDB database. The code works perfectly in local host, but when I run firebase deploy, I get an error. Here is the error message: "Function failed on loading user code. This is likely due to a bug in the user code. Error message: Error: please examine your function logs to see the error cause: https://cloud.google.com/functions/docs/monitoring/logging#viewing_logs. Additional troubleshooting documentation can be found at https://cloud.google.com/functions/docs/troubleshooting#logging. Please visit https://cloud.google.com/functions/docs/troubleshooting for in-depth troubleshooting documentation."}"

I know that the problem consists of these two lines:

const pup = require("puppeteer"); 


const WebsiteData = require('./schema');  

When I comment out those two lines I can deploy the function. Here is the code for schema.js:

var mongoose = require('mongoose')
mongoose.connect("URI");

var Schema = mongoose.Schema

var websiteData = new Schema({
    sites: [{
        name: {
            type: String,
            required : true,
        },
        nDUniqueUsers: {
            type: String,
            required : true,
        },
        nDTransactions: {
            type: String,
            required : true,
        }, fee: {
            type: String,
            required : true,
        }, url: {
            type: String,
            required : true,
        }

    }],
})

var WebsiteData = mongoose.model("SiteData", websiteData);

module.exports = WebsiteData

I do not know how to fix this. Any help would be greatly appreciated.

CodePudding user response：

I suspect you haven't included "puppeteer" in the dependencies section of the package.json deployed alongside your function. Or possibly you've inadvertently included it in the devDependencies section instead of dependencies.

// package.json
{
  // ...
  "dependencies": {
    "puppeteer": "^13.5.1" // be sure you have this
  }
}