The rundown:

I'm running the below function in an await Promise.all() in an outer wrapper function. The function essentially does some internet scraping and then adds rows to a Postgres table. I call the function 3 times, with a pagination offset to hit all the results I need to.

The problem:

It's adding the correct amount of rows, but there's clearly some variable confusion, because the output in the table has duplicate data sometimes (and the duplicated data varies from run to run). I think it's because during each iteration in the for loop, it might be referencing the data in memory from one of the other concurrently running functions.

The code:

const functionName = function (object, timestamp) {
  new Promise(resolve => {
    search.json(object, function (data) {
      resolve(data);
    })
  }).then(data => async function () {
    const client = await pool.connect();
    try {
      await client.query('BEGIN');
      let results = data.results
      if (results != null) {
        for (result of results) {
          let type = "item"
          let title = result.title
          var loop1 = result.loop1;
          var loop2 = result.loop2;
          let expiration = timestamp
          var time = result.time;

          await client.query(`INSERT INTO tableName (type, title, loop1, loop2, expiration, time) VALUES($1, $2, $3, $4, $5, $6) ON CONFLICT DO NOTHING`, [type, title, loop1, loop2, expiration, time]);
        }
      } else {
        console.log("No results")
      }
      await client.query('COMMIT');
    } catch (err) {
      console.log(err)
      await client.query('ROLLBACK');
    }
  }());
};

How can I run the same function in parallel without the variables inside one getting confused for the other concurrent runs?

CodePudding user response：

for (result of results) { is missing a variable declaration. Always use strict mode to avoid the horror of implicit globals!

Also that arrow returning an AIIFE being passed to then is very weird. Simplify to

async function functionName { /*
^^^^^^ */
  const data = await new Promise(resolve => {
//             ^^^^^
    search.json(object, resolve);
  });
  const client = await pool.connect();
  try {
    await client.query('BEGIN');
    const {results} = data;
    if (results != null) {
      for (const result of results) {
//         ^^^^^
        const type = "item"
        const {title, loop1, loop2, time} = result;
        const expiration = timestamp;

        await client.query(`INSERT INTO tableName (type, title, loop1, loop2, expiration, time) VALUES($1, $2, $3, $4, $5, $6) ON CONFLICT DO NOTHING`, [type, title, loop1, loop2, expiration, time]);
      }
    } else {
      console.log("No results")
    }
    await client.query('COMMIT');
  } catch (err) {
    console.log(err)
    await client.query('ROLLBACK');
  }
}