I have a javascript web scraping program for stock prices that I want to run automatically on netlify.

In order for it to run I have to type in "node index.js" in the command terminal.

But.. how can I make it do so AUTOMATICALLY after uploading it to netlify - without maually typing "node index.js"? Web Scraping/API Practice

It's my first web scraping project - so please be understanding - it's probably a VERRY noobie question.

Here's the index.js file

const request = require("request-promise");
const fs = require("fs");
const cheerio = require("cheerio");


async function tsla() {
    const html = await request.get(
        "https://finance.yahoo.com/quote/TSLA/"
        );

    const $ = await cheerio.load(html);

    const Price = $('[]').text();
    const Symbol = $('[]').text();
    const PercentChange = $('fin-streamer[][data-field="regularMarketChangePercent"]').find('span').text();
    fs.writeFileSync("./tslaSymbol.csv", Symbol);
    fs.writeFileSync("./tslaPrice.csv", Price);
    fs.writeFileSync("./tslaPercentChange.csv", PercentChange)
    
    console.log(Price);
    console.log(Symbol);
    console.log(PercentChange)

    setTimeout(tsla, 1000); //60 seconds == 1minute
}

tsla();

async function aapl() {
    const html = await request.get(
        "https://finance.yahoo.com/quote/AAPL/"
        );

    const $ = await cheerio.load(html);

    const Price = $('[]').text();
    const Symbol = $('[]').text();
    const PercentChange = $('fin-streamer[][data-field="regularMarketChangePercent"]').find('span').text();
    fs.writeFileSync("./aaplSymbol.csv", Symbol);
    fs.writeFileSync("./aaplPrice.csv", Price);
    fs.writeFileSync("./aaplPercentChange.csv", PercentChange)
    
    
    console.log(Price);
    console.log(Symbol);

    setTimeout(aapl, 1000); // 60,000 = 60 seconds == 1minute
}

aapl();


async function spy() {
    const html = await request.get(
        "https://finance.yahoo.com/quote/spy/"
        );

    const $ = await cheerio.load(html);

    const Price = $('[]').text();
    const Symbol = $('[]').text();
    const PercentChange = $('fin-streamer[][data-field="regularMarketChangePercent"]').find('span').text();
    fs.writeFileSync("./spySymbol.csv", Symbol);
    fs.writeFileSync("./spyPrice.csv", Price);
    fs.writeFileSync("./spyPercentChange.csv", PercentChange)
    
    console.log(Price);
    console.log(Symbol);
    console.log(PercentChange);

    setTimeout(spy, 1000); // 60,000 = 60 seconds == 1minute
}

spy();

async function nvda() {
    const html = await request.get(
        "https://finance.yahoo.com/quote/nvda/"
        );

    const $ = await cheerio.load(html);

    const Price = $('[]').text();
    const Symbol = $('[]').text();
    const PercentChange = $('fin-streamer[][data-field="regularMarketChangePercent"]').find('span').text();
    fs.writeFileSync("./nvdaSymbol.csv", Symbol);
    fs.writeFileSync("./nvdaPrice.csv", Price);
    fs.writeFileSync("./nvdaPercentChange.csv", PercentChange)
    
    console.log(Price);
    console.log(Symbol);
    console.log(PercentChange);

    setTimeout(nvda, 1000); // 60,000 = 60 seconds == 1minute
}

nvda();

Basically, the javascript scrapes that data and sends it to a CSV file which then is displayed into the page.

CodePudding user response：

I think you will have to create two files in your project before your deploy it to netlify,

1. netlify.toml (where you would be putting your build function)
2. you will have to create a dist dir with an empty index.html file.

if you knew it, just ignore that and make sure you have written your scripts in package.json,then while deploying you should be able to configure you build command, with that it should be all fine.

you can get some help from here too netifly functions are the ways to do since netlify hosts static compatible