How can I check an array of links to see if they work and write them to a file?-CodePudding

I have an array consisting of 40,000 company names.

That is, there is such a link on the site https://numfin.com/uk/AAPL/overview

in my case AAPL, this is the name of the company.

I decided to accomplish this task by simply writing these 40k company names into an array, and in a loop just change the company name in the link

I wrote a small script on node js in which I made a loop and changed the names of the companies

import request from "request";
import fs from "fs";
import { tickets } from "./tickets.js"; // array of company names

var data = [];

var out = fs.createWriteStream("tickets.txt", { flags: "a" });

for (let i = 0; i < tickets.length; i  ) {
  request(
    `https://numfin.com/uk/${tickets[i]}/overview`,
    function (error, response, body) {
      if (error) {
        console.log("Err: "   error);
        return false;
      }

      if (
        response.statusCode == 200 ||
        response.statusCode == 201 ||
        response.statusCode == 202
      ) {
        console.log(
          `https://numfin.com/uk/${tickets[i]}/overview`   " is alive!!"
        );

        return false;
      }

      if (response.statusCode == 301 || response.statusCode == 302) {
        console.log(
          `https://numfin.com/uk/${tickets[i]}/overview`  
            " is redirecting us!!"
        );
        return false;
      }

      if (response.statusCode == 401) {
        console.log(
          "you are unauthorized to "  
            `https://numfin.com/uk/${tickets[i]}/overview`
        );
        return false;
      } else {
        // IF TICKET === 500 status code, I will write this ticket to file
        console.log(
          `https://numfin.com/uk/${tickets[i]}/overview`   " is dead!!"
        );
        data.push(tickets[i]);
        console.log("data length", data.length);
        out.write(tickets[i].toString());
      }
    }
  );
}

out.end();

I wrote a small script on node js in which I made a loop and changed the names of the companies.

But the problem is that this code does not work and does not write to the file.

And for some reason, when I stuff such a large volume of data, I get the following error:
Error: read ECONNRESET

I'm already tired of trying to solve it, tell me maybe some other options

CodePudding user response：

It looks like your code is making a large number of requests to the server in a short period of time, which may be causing the server to close the connection. One way to avoid this issue is to use a library like async or promise to make the requests in a more controlled manner.

For example, you could use the async library to create a function that makes a request and returns the response status code. You can then use the map function to create an array of promises that represent the requests, and use the Promise.all() function to wait for all of the requests to complete before writing the results to the file.

Here is an example of how you could modify your code to use the async library:

import request from "request";
import fs from "fs";
import async from "async";
import { tickets } from "./tickets.js"; // array of company names

var data = [];

var out = fs.createWriteStream("tickets.txt", { flags: "a" });

async function checkUrl(url) {
  return new Promise((resolve, reject) => {
    request(url, function (error, response, body) {
      if (error) {
        console.log("Err: "   error);
        resolve(false);
      }

      if (
        response.statusCode == 200 ||
        response.statusCode == 201 ||
        response.statusCode == 202
      ) {
        console.log(`${url} is alive!!`);
        resolve(false);
      }

      if (response.statusCode == 301 || response.statusCode == 302) {
        console.log(`${url} is redirecting us!!`);
        resolve(false);
      }

      if (response.statusCode == 401) {
        console.log(`You are unauthorized to ${url}`);
        resolve(false);
      } else {
        console.log(`${url} is dead!!`);
        resolve(true);
      }
    });
  });
}

async function main() {
  const requests = tickets.map((ticket) => {
    const url = `https://numfin.com/uk/${ticket}/overview`;
    return checkUrl(url);
  });

  const results = await Promise.all(requests);

  for (let i = 0; i < results.length; i  ) {
    if (results[i]) {
      data.push(tickets[i]);
      out.write(tickets[i].toString());
    }
  }

  out.end();
}

main();