Deduplication not working properly even after redis implementation-CodePudding

I am storing jobdid in redis onclickCache.setex to see if job is already present for an email which is being sent to the checkForDuplicate function. In this function i check from this redis function onclickCache.get to see list of jobdid already present for an email and if not then i insert for that email using onclickCache.setex function.

Issue I am still getting duplicate applications even after implementation of this code

var message = {
  "jobdid": "JCLT",
  "email": "[email protected]"
}

async function checkForDuplicate(message){
  try {
  let  jsnMessage = {email:message.email.toLowerCase(),jobdid:message.jobdid.toLowerCase()};
    let listJobApps = await onclickCache.get(jsnMessage.email);

    if(listJobApps){
        jobArr = listJobApps.jobdids;
         let duplicate = jobArr.filter(job => job==jsnMessage.jobdid);

         if(duplicate.length > 0){
            return false;
         }
         else{
            jobArr.push(`${jsnMessage.jobdid}`);
            await onclickCache.setex(jsnMessage.email,JSON.stringify({jobdids:[`${jsnMessage.jobdid}`]}));
         }
    }
    else{
        await onclickCache.setex(jsnMessage.email,JSON.stringify({jobdids:[`${jsnMessage.jobdid}`]}));
    }
  } catch (error) {
    console.log("Check for duplication failed",error); 
  }
  return true;
}

const expireylimit = 60 * 60 * 6;

setex method

setex(key, value) {
    return new Promise((resolve, reject) => {
        this.client.setex(key, expireylimit, value, (error, result) => {
            if (error) {
                console.log("Failed to set value for", key, value);
                console.log(error);
                return reject(new Error(error));
            }
            return resolve(result);
        });
    });
}

get method

get(key) {
    return new Promise((resolve, reject) => {
        this.client.get(key, (error, result) => {
            if (error) {
                console.log("Failed to get value for", key);
                console.log(error);
                return reject(new Error(error));
            }
            try {
                const parsedData = JSON.parse(result);
                return resolve(parsedData);
            } catch (e) {
                return reject(new Error(e));
            }
            
        });
    });
}

CodePudding user response：

Looks like you're looking for setnx, not setex. I'd put each email's jobs in a hash of their own. (This is assuming you're using a promiseful Redis library such as ioredis.)

async function checkForDuplicate(message) {
  const email = message.email.toLowerCase();
  const jobdid = message.jobdid.toLowerCase();
  const res = await redis.hsetnx(`jobs:${email}`, jobdid,  new Date());
  if (res === 0) {
    // duplicate jobdid
    return null;
  }
  // jobdid was added to this email's jobs hash
  return jobdid;
}

async function getJobdids(message) {
  const email = message.email.toLowerCase();
  return redis.hkeys(`jobs:${email}`); // Promise of strings
}

CodePudding user response：

This looks incorrect - you are only ever storing the single job-id and overwriting previous ones.

    if(listJobApps){
        jobArr = listJobApps.jobdids;
         let duplicate = jobArr.filter(job => job==jsnMessage.jobdid);

         if(duplicate.length > 0){
            return false;
         }
         else{
            jobArr.push(`${jsnMessage.jobdid}`);
            await onclickCache.setex(jsnMessage.email,JSON.stringify({jobdids:[`${jsnMessage.jobdid}`]}));
         }
    }

It should be

await onclickCache.setex(jsnMessage.email,JSON.stringify({jobdids:[`${jobArr}`]}));