I've this Twitter bot that tweets out info on stock trades, whenever I run the bot from VS Code it works fine and tweets the values I want. Now I need to host it on Heroku but when I do and the script executes the values that are tweeted end up as undefined, see image. This is happening to priceModule and nameModule and I don't understand why, its working perfectly locally, Is there a fix for this?
Index.js
// MODULES
const rwClient = require("./TwitterClient.js");
const cronjob = require("cron").CronJob;
const priceModule = require("./price");
const nameModule = require("./name");
(async () => {
// Async function that creates the Tweet
const tweet = async () => {
try {
await rwClient.v2.tweet(
"Name: " await nameModule() '\n'
"Amount Purchased: " await priceModule() '\n'
);
} catch (error) {
console.error(error)
}
}
console.log(
"Name: " await nameModule() '\n'
"Amount Purchased: " await priceModule() '\n'
);
tweet();
console.log("Tweet executed");
// CronJob, executes every 6 hours
const job = new cronjob("0 */4 * * *", () => {
tweet();
console.log("Next tweet executed");
});
job.start();
})();
nameModule.js
// MODULES
const puppeteer = require("puppeteer");
// Url where we get and scrape the data from
const url = "https://www.sec.gov/edgar/search/#/dateRange=30d&category=custom&forms=4";
let browser;
module.exports = () => (async () => {
browser = await puppeteer.launch();
const [page] = await browser.pages();
const $ = (...args) => page.waitForSelector(...args);
const text = async (...args) =>
(await $(...args)).evaluate(el => el.textContent.trim());
await page.goto(url, {waitUntil: "domcontentloaded"});
await page.reload({waitUntil: "domcontentloaded"});
const info = {
secTableEN: await text(".table td.entity-name"),
secTableFiled: await text(".table td.filed"),
secTableLink: await text(".table td.filetype"),
};
return info.secTableEN;
})()
.catch(err => console.error(err))
.finally(() => browser?.close());
priceModule
// MODULES
const puppeteer = require("puppeteer");
// Url where we get and scrape the data from
const url = "https://www.sec.gov/edgar/search/#/dateRange=30d&category=custom&forms=4";
let browser;
module.exports = () => (async () => {
browser = await puppeteer.launch();
const [page] = await browser.pages();
const ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36";
await page.setUserAgent(ua);
await page.goto(url, {waitUntil: "domcontentloaded", timeout: 0});
await page.reload({waitUntil: "domcontentloaded"});
const responseP = page.waitForResponse(res =>
res.status() === 200 && res.url().endsWith(".xml")
);
const a = await page.waitForSelector(".filetype .preview-file");
await a.click();
const html = await (await responseP).text();
await page.evaluate(html => document.body.outerHTML = html, html);
const price = await page.$$eval(".FormText", els =>
els.find(e => e.textContent.trim() === "$")
.parentNode
.textContent
.trim()
);
return price;
})()
.catch(err => console.error(err))
.finally(() => browser?.close());
Activity log from Heroku
2022-11-15T12:44:00.060455 00:00 app[worker.1]: at ChromeLauncher.executablePath (/app/node_modules/puppeteer-core/lib/cjs/puppeteer/node/ChromeLauncher.js:166:25)
2022-11-15T12:44:00.060455 00:00 app[worker.1]: at ChromeLauncher.launch (/app/node_modules/puppeteer-core/lib/cjs/puppeteer/node/ChromeLauncher.js:70:37)
2022-11-15T12:44:00.060455 00:00 app[worker.1]: at async /app/numShares.js:9:15
2022-11-15T12:44:00.060456 00:00 app[worker.1]: at async CJ.<anonymous> (/app/index.js:39:46)
2022-11-15T12:44:00.060711 00:00 app[worker.1]: Error: Could not find Chromium (rev. 1056772). This can occur if either
2022-11-15T12:44:00.060711 00:00 app[worker.1]: 1. you did not perform an installation before running the script (e.g. `npm install`) or
2022-11-15T12:44:00.060711 00:00 app[worker.1]: 2. your cache path is incorrectly configured (which is: /app/.cache/puppeteer).
2022-11-15T12:44:00.060712 00:00 app[worker.1]: For (2), check out our guide on configuring puppeteer at https://pptr.dev/guides/configuration.
2022-11-15T12:44:00.060712 00:00 app[worker.1]: at ChromeLauncher.resolveExecutablePath (/app/node_modules/puppeteer-core/lib/cjs/puppeteer/node/ProductLauncher.js:120:27)
2022-11-15T12:44:00.060712 00:00 app[worker.1]: at ChromeLauncher.executablePath (/app/node_modules/puppeteer-core/lib/cjs/puppeteer/node/ChromeLauncher.js:166:25)
2022-11-15T12:44:00.060713 00:00 app[worker.1]: at ChromeLauncher.launch (/app/node_modules/puppeteer-core/lib/cjs/puppeteer/node/ChromeLauncher.js:70:37)
2022-11-15T12:44:00.060713 00:00 app[worker.1]: at async /app/price.js:9:15
2022-11-15T12:44:00.060713 00:00 app[worker.1]: at async CJ.<anonymous> (/app/index.js:39:66)
2022-11-15T12:44:00.060953 00:00 app[worker.1]: Error: Could not find Chromium (rev. 1056772). This can occur if either
2022-11-15T12:44:00.060954 00:00 app[worker.1]: 1. you did not perform an installation before running the script (e.g. `npm install`) or
2022-11-15T12:44:00.060954 00:00 app[worker.1]: 2. your cache path is incorrectly configured (which is: /app/.cache/puppeteer).
2022-11-15T12:44:00.060954 00:00 app[worker.1]: For (2), check out our guide on configuring puppeteer at https://pptr.dev/guides/configuration.
2022-11-15T12:44:00.060954 00:00 app[worker.1]: at ChromeLauncher.resolveExecutablePath (/app/node_modules/puppeteer-core/lib/cjs/puppeteer/node/ProductLauncher.js:120:27)
2022-11-15T12:44:00.060955 00:00 app[worker.1]: at ChromeLauncher.executablePath (/app/node_modules/puppeteer-core/lib/cjs/puppeteer/node/ChromeLauncher.js:166:25)
2022-11-15T12:44:00.060955 00:00 app[worker.1]: at ChromeLauncher.launch (/app/node_modules/puppeteer-core/lib/cjs/puppeteer/node/ChromeLauncher.js:70:37)
2022-11-15T12:44:00.060955 00:00 app[worker.1]: at async /app/stock.js:9:15
2022-11-15T12:44:00.060955 00:00 app[worker.1]: at async CJ.<anonymous> (/app/index.js:40:30)
2022-11-15T12:44:00.061194 00:00 app[worker.1]: Error: Could not find Chromium (rev. 1056772). This can occur if either
2022-11-15T12:44:00.061195 00:00 app[worker.1]: 1. you did not perform an installation before running the script (e.g. `npm install`) or
2022-11-15T12:44:00.061195 00:00 app[worker.1]: 2. your cache path is incorrectly configured (which is: /app/.cache/puppeteer).
2022-11-15T12:44:00.061195 00:00 app[worker.1]: For (2), check out our guide on configuring puppeteer at https://pptr.dev/guides/configuration.
2022-11-15T12:44:00.061195 00:00 app[worker.1]: at ChromeLauncher.resolveExecutablePath (/app/node_modules/puppeteer-core/lib/cjs/puppeteer/node/ProductLauncher.js:120:27)
2022-11-15T12:44:00.061196 00:00 app[worker.1]: at ChromeLauncher.executablePath (/app/node_modules/puppeteer-core/lib/cjs/puppeteer/node/ChromeLauncher.js:166:25)
2022-11-15T12:44:00.061196 00:00 app[worker.1]: at ChromeLauncher.launch (/app/node_modules/puppeteer-core/lib/cjs/puppeteer/node/ChromeLauncher.js:70:37)
2022-11-15T12:44:00.061196 00:00 app[worker.1]: at async /app/date.js:9:13
2022-11-15T12:44:00.061197 00:00 app[worker.1]: at async CJ.<anonymous> (/app/index.js:41:28)
2022-11-15T12:44:00.061227 00:00 app[worker.1]: New insider trade! (form 4 filed)
2022-11-15T12:44:00.061227 00:00 app[worker.1]:
2022-11-15T12:44:00.061228 00:00 app[worker.1]: undefined bought undefined shares at $undefined
2022-11-15T12:44:00.061228 00:00 app[worker.1]:
2022-11-15T12:44:00.061228 00:00 app[worker.1]: Amount Purchased: $NaN
2022-11-15T12:44:00.061229 00:00 app[worker.1]: Stock: undefined
2022-11-15T12:44:00.061229 00:00 app[worker.1]: Date: undefined
2022-11-15T12:44:00.061229 00:00 app[worker.1]:
2022-11-15T12:44:01.000000 00:00 app[api]: Build started by user [email protected]
2022-11-15T12:44:38.942638 00:00 app[api]: Release v8 created by user [email protected]
2022-11-15T12:44:38.942638 00:00 app[api]: Deploy 2db65223 by user [email protected]
2022-11-15T12:44:40.624181 00:00 heroku[worker.1]: Restarting
2022-11-15T12:44:40.626042 00:00 heroku[worker.1]: State changed from up to starting
2022-11-15T12:44:39.000000 00:00 app[api]: Build succeeded
2022-11-15T12:44:41.596325 00:00 heroku[worker.1]: Stopping all processes with SIGTERM
2022-11-15T12:44:41.950594 00:00 heroku[worker.1]: Process exited with status 143
2022-11-15T12:44:44.285628 00:00 heroku[worker.1]: Starting process with command `node index.js`
2022-11-15T12:44:45.072715 00:00 heroku[worker.1]: State changed from starting to up
2022-11-15T12:44:53.176410 00:00 heroku[worker.1]: Restarting
2022-11-15T12:44:53.191437 00:00 heroku[worker.1]: State changed from up to starting
2022-11-15T12:44:54.322495 00:00 heroku[worker.1]: Stopping all processes with SIGTERM
2022-11-15T12:44:54.589561 00:00 heroku[worker.1]: Process exited with status 143
2022-11-15T12:44:55.711588 00:00 heroku[worker.1]: Starting process with command `node index.js`
2022-11-15T12:44:56.492554 00:00 heroku[worker.1]: State changed from starting to up
CodePudding user response：
As the logs clearly states - it's not a problem with twitter bot but rather puppeteer over Heroku
Why? Because puppeteer is a complex module that requires customized builds during installation. Your hosted machine is limited (in terms of permissions) WAY more than your local machine - and that's why you cant really pin-point the problem from your own machine
Luckily for you, both Heroku and puppeteer are commonly used solutons and that is why there is a buildpack called puppeteer-heroku-buildpack that does just that.
I recommend that you follow these steps and give it a spin
Edit:
From your logs it also seems that chromium is not properly configured. Please note that using Heroku requires some configuration (it is NOT as chrome headless buildpack) that can be found here
Ensure to perform all these prerequisites and only then install puppeteer
CodePudding user response：
I would recommend using a different API, given Chris's diligent reading of SEC Terms and Services. Alphavantage looks like a good, simple, free API to use for your needs:
Cheers!