remove duplicate data from large data api in flutter-CodePudding

i want to remove duplicate data from the array that i am getting from api and i have tried sorting, compairing, toSet().toList() but nothing seems to work. below is the data that i am getitng -:

{
  "data":[
       {
      "laboratoryComponentId": 16,
      "laboratoryTypeId": 18,
      "laboratoryTypeName": "Profile1",
      "componentName": "LP-PLA2 Enzyme",
      "uom": "U/L",
      "componentDataType": "Numeric",
      "componentDataTypeValue": null,
      "laboratoryTypeSequence": 1,
      "laboratoryComponentSequence": 16
    },
    {
      "laboratoryComponentId": 17,
      "laboratoryTypeId": 18,
      "laboratoryTypeName": "Profile1",
      "componentName": "CRP C Reactive Protein",
      "uom": "mg/dl",
      "componentDataType": "Numeric",
      "componentDataTypeValue": null,
      "laboratoryTypeSequence": 1,
      "laboratoryComponentSequence": 17
    },
    {
      "laboratoryComponentId": 18,
      "laboratoryTypeId": 25,
      "laboratoryTypeName": "Profile2",
      "componentName": "Anti TPO (Anti Micro Somal) Antibody",
      "uom": "IU/ML",
      "componentDataType": "Numeric",
      "componentDataTypeValue": null,
      "laboratoryTypeSequence": 2,
      "laboratoryComponentSequence": 18
    },
    {
      "laboratoryComponentId": 19,
      "laboratoryTypeId": 25,
      "laboratoryTypeName": "Profile2",
      "componentName": "FT3",
      "uom": "pg/ml",
      "componentDataType": "Numeric",
      "componentDataTypeValue": null,
      "laboratoryTypeSequence": 2,
      "laboratoryComponentSequence": 19
    },
    {
      "laboratoryComponentId": 30,
      "laboratoryTypeId": 8,
      "laboratoryTypeName": "Profile3",
      "componentName": "Fg3",
      "uom": "pg/ml",
      "componentDataType": "Numeric",
      "componentDataTypeValue": null,
      "laboratoryTypeSequence": 3,
      "laboratoryComponentSequence": 30
    },
  
   ]
}

here i want to make 2 list one for "laboratoryTypeName" and other for "componentName" . can anyone help how i remove duplicate data us add it to a object so that i can used all data when needed. thanks

EDIT -: below is the model class-

import 'dart:convert';

GetLaboratorComponents getLaboratorComponentsFromJson(String str) => GetLaboratorComponents.fromJson(json.decode(str));

String getLaboratorComponentsToJson(GetLaboratorComponents data) => json.encode(data.toJson());

class GetLaboratorComponents {
  GetLaboratorComponents({
    this.data,
    this.exceptionInfo,
    this.message,
    this.messages,
    this.isSuccess,
  });

  List<LaboratorComponents> data;
  dynamic exceptionInfo;
  dynamic message;
  dynamic messages;
  bool isSuccess;

  factory GetLaboratorComponents.fromJson(Map<String, dynamic> json) => GetLaboratorComponents(
    data: List<LaboratorComponents>.from(json["data"].map((x) => LaboratorComponents.fromJson(x))),
    exceptionInfo: json["exceptionInfo"],
    message: json["message"],
    messages: json["messages"],
    isSuccess: json["isSuccess"],
  );

  Map<String, dynamic> toJson() => {
    "data": List<dynamic>.from(data.map((x) => x.toJson())),
    "exceptionInfo": exceptionInfo,
    "message": message,
    "messages": messages,
    "isSuccess": isSuccess,
  };
}

class LaboratorComponents {
  LaboratorComponents({
    this.laboratoryComponentId,
    this.laboratoryTypeId,
    this.laboratoryTypeName,
    this.componentName,
    this.uom,
    this.componentDataType,
    this.componentDataTypeValue,
    this.laboratoryTypeSequence,
    this.laboratoryComponentSequence,
  });

  int laboratoryComponentId;
  int laboratoryTypeId;
  String laboratoryTypeName;
  String componentName;
  String uom;
  dynamic componentDataType;
  String componentDataTypeValue;
  int laboratoryTypeSequence;
  int laboratoryComponentSequence;

  factory LaboratorComponents.fromJson(Map<String, dynamic> json) => LaboratorComponents(
    laboratoryComponentId: json["laboratoryComponentId"],
    laboratoryTypeId: json["laboratoryTypeId"],
    laboratoryTypeName: json["laboratoryTypeName"],
    componentName: json["componentName"],
    uom: json["uom"] == null ? null : json["uom"],
    componentDataType: json["componentDataType"],
    componentDataTypeValue: json["componentDataTypeValue"] == null ? null : json["componentDataTypeValue"],
    laboratoryTypeSequence: json["laboratoryTypeSequence"],
    laboratoryComponentSequence: json["laboratoryComponentSequence"],
  );

  Map<String, dynamic> toJson() => {
    "laboratoryComponentId": laboratoryComponentId,
    "laboratoryTypeId": laboratoryTypeId,
    "laboratoryTypeName": laboratoryTypeName,
    "componentName": componentName,
    "uom": uom == null ? null : uom,
    "componentDataType": componentDataType,
    "componentDataTypeValue": componentDataTypeValue == null ? null : componentDataTypeValue,
    "laboratoryTypeSequence": laboratoryTypeSequence,
    "laboratoryComponentSequence": laboratoryComponentSequence,
  };
}

CodePudding user response：

You can use removeWhere to find duplicate elements and remove it. I used two keys (typeName and Id).

GetLaboratorComponents labModel;

List<LaboratorComponents> dataList = [];

labModel = GetLaboratorComponents.fromJson(response);

labModel.data.forEach((element) {
  dataList.removeWhere((e) => element.laboratoryTypeName == e.laboratoryTypeName || element.laboratoryComponentId == e.laboratoryComponentId);
  dataList.add(element);
});

labModel.data = dataList;

return labModel;

CodePudding user response：

The following example code creates a set for "laboratoryTypeName" and "componentName":

void main() {
  const data = {
    "data": [
      {
        "laboratoryComponentId": 16,
        "laboratoryTypeId": 18,
        "laboratoryTypeName": "Profile1",
        "componentName": "LP-PLA2 Enzyme",
        "uom": "U/L",
        "componentDataType": "Numeric",
        "componentDataTypeValue": null,
        "laboratoryTypeSequence": 1,
        "laboratoryComponentSequence": 16
      },
      {
        "laboratoryComponentId": 17,
        "laboratoryTypeId": 18,
        "laboratoryTypeName": "Profile1",
        "componentName": "CRP C Reactive Protein",
        "uom": "mg/dl",
        "componentDataType": "Numeric",
        "componentDataTypeValue": null,
        "laboratoryTypeSequence": 1,
        "laboratoryComponentSequence": 17
      },
      {
        "laboratoryComponentId": 18,
        "laboratoryTypeId": 25,
        "laboratoryTypeName": "Profile2",
        "componentName": "Anti TPO (Anti Micro Somal) Antibody",
        "uom": "IU/ML",
        "componentDataType": "Numeric",
        "componentDataTypeValue": null,
        "laboratoryTypeSequence": 2,
        "laboratoryComponentSequence": 18
      },
      {
        "laboratoryComponentId": 19,
        "laboratoryTypeId": 25,
        "laboratoryTypeName": "Profile2",
        "componentName": "FT3",
        "uom": "pg/ml",
        "componentDataType": "Numeric",
        "componentDataTypeValue": null,
        "laboratoryTypeSequence": 2,
        "laboratoryComponentSequence": 19
      },
      {
        "laboratoryComponentId": 30,
        "laboratoryTypeId": 8,
        "laboratoryTypeName": "Profile3",
        "componentName": "Fg3",
        "uom": "pg/ml",
        "componentDataType": "Numeric",
        "componentDataTypeValue": null,
        "laboratoryTypeSequence": 3,
        "laboratoryComponentSequence": 30
      },
    ]
  };

  final Set<String> laboratoryTypeNames = {
    for (final item in data['data'] ?? []) item['laboratoryTypeName'],
  };

  final Set<String> componentNames = {
    for (final item in data['data'] ?? []) item['componentName'],
  };

  print(laboratoryTypeNames);
  print(componentNames);
}

Here is an example of how you would add this to your model. I would add getters on GetLaboratorComponents that return the Set objects.

(also, I had to update the code slightly because you appear to be using an outdated version of dart and your code would not compile in the latest dart version).

import 'dart:convert';

GetLaboratorComponents getLaboratorComponentsFromJson(String str) =>
    GetLaboratorComponents.fromJson(json.decode(str));

String getLaboratorComponentsToJson(GetLaboratorComponents data) =>
    json.encode(data.toJson());

class GetLaboratorComponents {
  GetLaboratorComponents({
    required this.data,
    required this.exceptionInfo,
    required this.message,
    required this.messages,
    required this.isSuccess,
  });

  List<LaboratorComponents> data;
  dynamic exceptionInfo;
  dynamic message;
  dynamic messages;
  bool? isSuccess;

  /*
   * Added getters for laboratoryTypeNames and componentNames here.
   */
  Set<String> get laboratoryTypeNames => {
        for (final item in data)
          if (item.laboratoryTypeName != null) item.laboratoryTypeName!,
      };

  Set<String> get componentNames => {
        for (final item in data)
          if (item.componentName != null) item.componentName!,
      };

  factory GetLaboratorComponents.fromJson(Map<String, dynamic> json) =>
      GetLaboratorComponents(
        data: List<LaboratorComponents>.from(
            json["data"].map((x) => LaboratorComponents.fromJson(x))),
        exceptionInfo: json["exceptionInfo"],
        message: json["message"],
        messages: json["messages"],
        isSuccess: json["isSuccess"],
      );

  Map<String, dynamic> toJson() => {
        "data": List<dynamic>.from(data.map((x) => x.toJson())),
        "exceptionInfo": exceptionInfo,
        "message": message,
        "messages": messages,
        "isSuccess": isSuccess,
      };
}

class LaboratorComponents {
  LaboratorComponents({
    required this.laboratoryComponentId,
    required this.laboratoryTypeId,
    required this.laboratoryTypeName,
    required this.componentName,
    required this.uom,
    required this.componentDataType,
    required this.componentDataTypeValue,
    required this.laboratoryTypeSequence,
    required this.laboratoryComponentSequence,
  });

  int? laboratoryComponentId;
  int? laboratoryTypeId;
  String? laboratoryTypeName;
  String? componentName;
  String? uom;
  dynamic componentDataType;
  String? componentDataTypeValue;
  int? laboratoryTypeSequence;
  int? laboratoryComponentSequence;

  factory LaboratorComponents.fromJson(Map<String, dynamic> json) =>
      LaboratorComponents(
        laboratoryComponentId: json["laboratoryComponentId"],
        laboratoryTypeId: json["laboratoryTypeId"],
        laboratoryTypeName: json["laboratoryTypeName"],
        componentName: json["componentName"],
        uom: json["uom"] == null ? null : json["uom"],
        componentDataType: json["componentDataType"],
        componentDataTypeValue: json["componentDataTypeValue"] == null
            ? null
            : json["componentDataTypeValue"],
        laboratoryTypeSequence: json["laboratoryTypeSequence"],
        laboratoryComponentSequence: json["laboratoryComponentSequence"],
      );

  Map<String, dynamic> toJson() => {
        "laboratoryComponentId": laboratoryComponentId,
        "laboratoryTypeId": laboratoryTypeId,
        "laboratoryTypeName": laboratoryTypeName,
        "componentName": componentName,
        "uom": uom == null ? null : uom,
        "componentDataType": componentDataType,
        "componentDataTypeValue":
            componentDataTypeValue == null ? null : componentDataTypeValue,
        "laboratoryTypeSequence": laboratoryTypeSequence,
        "laboratoryComponentSequence": laboratoryComponentSequence,
      };
}

void main() {
  const data = {
    "data": [
      {
        "laboratoryComponentId": 16,
        "laboratoryTypeId": 18,
        "laboratoryTypeName": "Profile1",
        "componentName": "LP-PLA2 Enzyme",
        "uom": "U/L",
        "componentDataType": "Numeric",
        "componentDataTypeValue": null,
        "laboratoryTypeSequence": 1,
        "laboratoryComponentSequence": 16
      },
      {
        "laboratoryComponentId": 17,
        "laboratoryTypeId": 18,
        "laboratoryTypeName": "Profile1",
        "componentName": "CRP C Reactive Protein",
        "uom": "mg/dl",
        "componentDataType": "Numeric",
        "componentDataTypeValue": null,
        "laboratoryTypeSequence": 1,
        "laboratoryComponentSequence": 17
      },
      {
        "laboratoryComponentId": 18,
        "laboratoryTypeId": 25,
        "laboratoryTypeName": "Profile2",
        "componentName": "Anti TPO (Anti Micro Somal) Antibody",
        "uom": "IU/ML",
        "componentDataType": "Numeric",
        "componentDataTypeValue": null,
        "laboratoryTypeSequence": 2,
        "laboratoryComponentSequence": 18
      },
      {
        "laboratoryComponentId": 19,
        "laboratoryTypeId": 25,
        "laboratoryTypeName": "Profile2",
        "componentName": "FT3",
        "uom": "pg/ml",
        "componentDataType": "Numeric",
        "componentDataTypeValue": null,
        "laboratoryTypeSequence": 2,
        "laboratoryComponentSequence": 19
      },
      {
        "laboratoryComponentId": 30,
        "laboratoryTypeId": 8,
        "laboratoryTypeName": "Profile3",
        "componentName": "Fg3",
        "uom": "pg/ml",
        "componentDataType": "Numeric",
        "componentDataTypeValue": null,
        "laboratoryTypeSequence": 3,
        "laboratoryComponentSequence": 30
      },
    ]
  };

  final laborator = GetLaboratorComponents.fromJson(data);
  print(laborator.laboratoryTypeNames);
  print(laborator.componentNames);
}