I'm working on an importer to import a list of facilities from a json file into my database. The importing works. But when i added some extra conditions to update existing facilities (whenever i run the script again and there is updated data inside the json) it stopped working the way it originally did. The updating works if i only have 1 facility inside the json file but if i add more than one i get the following error:

Traceback (most recent call last):
  File "C:\json\jsontest\data_import\management\commands\import_from_url.py", line 53, in import_facility_from_file
    facility, facility_created = Facility.objects.update(
TypeError: cannot unpack non-iterable int object

Also: Instead of updating it will just create two additional copies of one of the facilities. Been trying to figure this out for a couple of days now but so far no luck.

UPDATE:

I changed it to what was suggested in the comments, (to use update_or_create instead of update) but it still does not work. It seems like now instead of producing an error or updating the facility it creates a copy of the facility.

import_facility_from_file.py

import os
import json
import traceback
from data_import.models import Facility, FacilityAddress, FacilityInspectionInfo, FacilityComplaints
from django.core.management.base import BaseCommand
from datetime import datetime
from jsontest.settings import BASE_DIR, STATIC_URL


class Command(BaseCommand):
    def import_facility_from_file(self):
        print(BASE_DIR)
        data_folder = os.path.join(BASE_DIR, 'import_data', 'resources')

        for data_file in os.listdir(data_folder):
            with open(os.path.join(data_folder, data_file), encoding='utf-8') as data_file:
                data = json.loads(data_file.read())

                for key, data_object in data.items():
                    UUID = key
                    Name = data_object.get('Name', None)
                    IssuedNumber = data_object.get('IssuedNumber', None)
                    Capacity = data_object.get('Capacity', None)
                    Licensee = data_object.get('Licensee', None)
                    Email = data_object.get('Email', None)
                    AdministratorName = data_object.get('AdministratorName', None)
                    TelephoneNumber = data_object.get('TelephoneNumber', None)
                    ClosedTimestamp = data_object.get('ClosedTimestamp', None)
                    MostRecentLicenseTimestamp = data_object.get('MostRecentLicenseTimestamp', None)
                    PrimaryAddress = data_object["AddressInfo"]["PrimaryAddress"]
                    SecondaryAddress = data_object["AddressInfo"]["SecondaryAddress"]
                    City = data_object["AddressInfo"]["City"]
                    RegionOrState = data_object["AddressInfo"]["RegionOrState"]
                    PostalCode = data_object["AddressInfo"]["PostalCode"]
                    Geolocation = data_object["AddressInfo"]["Geolocation"]
                    ComplaintRelatedVisits = data_object["InspectionInfo"]["ComplaintRelatedVisits"]
                    InspectionRelatedVisits = data_object["InspectionInfo"]["InspectionRelatedVisits"]
                    NumberOfVisits = data_object["InspectionInfo"]["NumberOfVisits"]
                    LastVisitTimestamp = data_object["InspectionInfo"]["LastVisitTimestamp"]
                    ComplaintsTypeA = data_object["Complaints"]["ComplaintsTypeA"]
                    ComplaintsTypeB = data_object["Complaints"]["ComplaintsTypeB"]
                    SubstantiatedAllegations = data_object["Complaints"]["SubstantiatedAllegations"]
                    TotalAllegations = data_object["Complaints"]["TotalAllegations"]
                    LatestUpdateTimestamp = data_object.get('LatestUpdateTimestamp', None)

                    try:
                        if Facility.objects.filter(UUID=UUID, LatestUpdateTimestamp=LatestUpdateTimestamp):
                            full_traceback = 'No update required: Facility exists and is up to date'
                            print(full_traceback)

                        else:
                            facility, facility_created = Facility.objects.update_or_create(UUID=UUID,
                                defaults={
                                'Name': Name,
                                'IssuedNumber': IssuedNumber,
                                'Capacity': Capacity,
                                'Licensee': Licensee,
                                'Email': Email,
                                'AdministratorName': AdministratorName,
                                'TelephoneNumber': TelephoneNumber,
                                'ClosedTimestamp': ClosedTimestamp,                                                                  
                                'MostRecentLicenseTimestamp': MostRecentLicenseTimestamp,
                                'LatestUpdateTimestamp': LatestUpdateTimestamp
                                }                            
                            ),
                            facility_address, address_created = FacilityAddress.objects.update_or_create(
                                defaults={
                                'PrimaryAddress': PrimaryAddress,
                                'SecondaryAddress': SecondaryAddress,
                                'City': City,
                                'RegionOrState': RegionOrState,
                                'PostalCode': PostalCode,
                                'Geolocation': Geolocation,
                                'AddressInfo': facility
                                }                            
                            )
                            if address_created:
                                    facility_address.save()
                          
                    except Exception:
                        full_traceback = traceback.format_exc()

                        print(full_traceback)


    
    def handle(self, *args, **options):
        """
        Call the function to import data
        """
        self.import_facility_from_file()

models.py

class Facility(models.Model):
    UUID = models.CharField(max_length=150, null=True, blank=True)
    Name = models.CharField(max_length=50, null=True, blank=True)
    admin_uid = models.OneToOneField(User, null=True, blank=True, on_delete=models.SET_NULL)
    IssuedNumber = models.CharField(max_length=20, null=True, blank=True)
    mainimage = models.ImageField(null=True, blank=True)
    Capacity = models.IntegerField(null=True, blank=True)
    Licensee = models.CharField(max_length=90, null=True, blank=True)
    Email = models.EmailField(max_length=100, null=True, blank=True)
    AdministratorName = models.CharField(max_length=30, null=True, blank=True)
    Status = models.CharField(max_length=10, null=True, blank=True)
    TelephoneNumber = models.CharField(max_length=20, null=True, blank=True)
    ClosedTimestamp = models.IntegerField(null=True, blank=True)
    MostRecentLicenseTimestamp = models.IntegerField(null=True, blank=True)
    LatestUpdateTimestamp = models.IntegerField(null=True, blank=True)

    class Meta:
        verbose_name_plural = "facilities"

    def __str__(self):
        return self.Name


class FacilityAddress(models.Model):
    PrimaryAddress = models.CharField(max_length=50, null=True, blank=True)
    SecondaryAddress = models.CharField(max_length=50, null=True, blank=True)
    City = models.CharField(max_length=50, null=True, blank=True)
    RegionOrState = models.CharField(max_length=30, null=True, blank=True)
    PostalCode = models.CharField(max_length=20, null=True, blank=True)
    Geolocation = models.CharField(max_length=20, null=True, blank=True)
    AddressInfo = models.ForeignKey(Facility, null=True, blank=True, on_delete=models.CASCADE)

    class Meta:
        verbose_name_plural = "facility addresses"

    
    def __str__(self):
        return f"{self.PrimaryAddress} {self.City}"


class FacilityInspectionInfo(models.Model):
    ComplaintRelatedVisits = models.IntegerField(null=True, blank=True)
    InspectionRelatedVisits = models.IntegerField(null=True, blank=True)
    NumberOfVisits = models.IntegerField(null=True, blank=True)
    LastVisitTimestamp = models.IntegerField(null=True, blank=True)
    InspectionInfo = models.ForeignKey(Facility, null=True, blank=True, on_delete=models.CASCADE)

    class Meta:
        verbose_name_plural = "facility inspection infos"

    def __str__(self):
        return self.InspectionInfo.Name


class FacilityComplaints(models.Model):
    ComplaintsTypeA = models.IntegerField(null=True, blank=True)
    ComplaintsTypeB = models.IntegerField(null=True, blank=True)
    SubstantiatedAllegations = models.IntegerField(null=True, blank=True)
    TotalAllegations = models.IntegerField(null=True, blank=True)
    Complaints = models.ForeignKey(Facility, null=True, blank=True, on_delete=models.CASCADE)

    class Meta:
        verbose_name_plural = "facility complaints"

    def __str__(self):
        return self.Complaints.Name

json file

{"00016ed7be4872a19d6e16afc98a7389b2bb324a2":
{"UUID":"00016ed7be4872a19d6e1ed6f36b647f3eb41cadedd2130b103a5851caebc26fbbbf24c2f1a64d2cf34ac4e03aaa30309816f58c397e6afc98a7389b2bb324a2","Name":"Test Facility","IssuedNumber":"123456","Licensee":"Test Licensee","Email":"[email protected]","AdministratorName":"Test Name","TelephoneNumber":"(123) 456-7890324879","LatestUpdateTimestamp":"1362985200",
    "AddressInfo":{"PrimaryAddress":"123 Fake Road","SecondaryAddress":"","City":"Testcity","RegionOrState":"TX","PostalCode":"12345","Geolocation":"00.0000,-00.0000"},"Capacity":100,"MostRecentLicenseTimestamp":1575180000,"ClosedTimestamp":0,
    "InspectionInfo":{"ComplaintRelatedVisits":0,"InspectionRelatedVisits":0,"NumberOfVisits":0,"LastVisitTimestamp":0},
    "Complaints":{"ComplaintsTypeA":0,"ComplaintsTypeB":0,"SubstantiatedAllegations":0,"TotalAllegations":0}},
    
"00016ed7be4872a15435435435b2bb324a2":
{"UUID":"000c93dcb7a0b3d5783bb330892aff6abdb9fb57a7d3701c2d903f3640877579f3173ecd8a80532f6c3d53dbacde78a6a54ae42fef321a5793f5a01934f8de7a","Name":"Test Facility 2","IssuedNumber":"123456","Licensee":"Test Licensee","Email":"[email protected]","AdministratorName":"Test Name","TelephoneNumber":"(123) 456-7890324879","LatestUpdateTimestamp":"1362985200",
    "AddressInfo":{"PrimaryAddress":"123 Fake Road","SecondaryAddress":"","City":"Testcity","RegionOrState":"TX","PostalCode":"12345","Geolocation":"00.0000,-00.0000"},"Capacity":100,"MostRecentLicenseTimestamp":1575180000,"ClosedTimestamp":0,
    "InspectionInfo":{"ComplaintRelatedVisits":0,"InspectionRelatedVisits":0,"NumberOfVisits":0,"LastVisitTimestamp":0},
    "Complaints":{"ComplaintsTypeA":0,"ComplaintsTypeB":0,"SubstantiatedAllegations":0,"TotalAllegations":0}},

"00234324324343243afc98a7389b2bb324a2":
{"UUID":"fffd4dec10054e6e1deb2a2266a7c6bb0136ba46222e734ceed5855651f735cfbe0bb66cfaf27c3d175ae261a8f6df0c36b5390d15c70b07d67e35e1081aaf6d","Name":"Test Facility 3","IssuedNumber":"123456","Licensee":"Test Licensee","Email":"[email protected]","AdministratorName":"Test Name","TelephoneNumber":"(123) 456-7890324879","LatestUpdateTimestamp":"1362985200",
    "AddressInfo":{"PrimaryAddress":"123 Fake Road","SecondaryAddress":"","City":"Testcity","RegionOrState":"TX","PostalCode":"12345","Geolocation":"00.0000,-00.0000"},"Capacity":100,"MostRecentLicenseTimestamp":1575180000,"ClosedTimestamp":0,
    "InspectionInfo":{"ComplaintRelatedVisits":0,"InspectionRelatedVisits":0,"NumberOfVisits":0,"LastVisitTimestamp":0},
    "Complaints":{"ComplaintsTypeA":0,"ComplaintsTypeB":0,"SubstantiatedAllegations":0,"TotalAllegations":0}}}

CodePudding user response：

Try the following if you want to create a new object if one is not found to be updated, or update it if it is. UUID=UUID would be the **kwargs used to find the object to be updated, otherwise one would be created. The defaults would be what you want to update the fields with. This is documented in https://docs.djangoproject.com/en/4.0/ref/models/querysets/#update-or-create.

I tried the code below, and it seems to work, although I'm not sure if this is what you want. I only did the Facility objects, not the others to see if it works. I removed the try/except and the exists() since I don't think you need them with the update_or_create() method. I also removed the get_or_create() since again, I think that is not needed since update_or_create() should take care of that as well.

class Command(BaseCommand):
    def import_facility_from_file(self):
        print("BASE", BASE_DIR)
        data_folder = os.path.join(BASE_DIR, 'import_data', 'resources')

        for data_file in os.listdir(data_folder):
            with open(os.path.join(data_folder, data_file), encoding='utf-8') as data_file:
                data = json.loads(data_file.read())

                for key, data_object in data.items():
                    UUID = data_object.get('UUID', None)
                    Name = data_object.get('Name', None)
                    IssuedNumber = data_object.get('IssuedNumber', None)
                    Capacity = data_object.get('Capacity', None)
                    Licensee = data_object.get('Licensee', None)
                    Email = data_object.get('Email', None)
                    AdministratorName = data_object.get('AdministratorName', None)
                    TelephoneNumber = data_object.get('TelephoneNumber', None)
                    ClosedTimestamp = data_object.get('ClosedTimestamp', None)
                    MostRecentLicenseTimestamp = data_object.get('MostRecentLicenseTimestamp', None)
                    PrimaryAddress = data_object["AddressInfo"]["PrimaryAddress"]
                    SecondaryAddress = data_object["AddressInfo"]["SecondaryAddress"]
                    City = data_object["AddressInfo"]["City"]
                    RegionOrState = data_object["AddressInfo"]["RegionOrState"]
                    PostalCode = data_object["AddressInfo"]["PostalCode"]
                    Geolocation = data_object["AddressInfo"]["Geolocation"]
                    ComplaintRelatedVisits = data_object["InspectionInfo"]["ComplaintRelatedVisits"]
                    InspectionRelatedVisits = data_object["InspectionInfo"]["InspectionRelatedVisits"]
                    NumberOfVisits = data_object["InspectionInfo"]["NumberOfVisits"]
                    LastVisitTimestamp = data_object["InspectionInfo"]["LastVisitTimestamp"]
                    ComplaintsTypeA = data_object["Complaints"]["ComplaintsTypeA"]
                    ComplaintsTypeB = data_object["Complaints"]["ComplaintsTypeB"]
                    SubstantiatedAllegations = data_object["Complaints"]["SubstantiatedAllegations"]
                    TotalAllegations = data_object["Complaints"]["TotalAllegations"]
                    LatestUpdateTimestamp = data_object.get('LatestUpdateTimestamp', None)


                    if Facility.objects.filter(LatestUpdateTimestamp=LatestUpdateTimestamp):
                        msg = "\n\nfacility exists and is up to date: {}\n{}".format(Name, str())
                        print(msg)

                    else:
                        print("UPDATE")
                        facility, facility_created = Facility.objects.update_or_create(UUID=UUID,
                            defaults={
                            'Name': Name,
                            'IssuedNumber': IssuedNumber,
                            'Capacity': Capacity,
                            'Licensee': Licensee,
                            'Email': Email,
                            'AdministratorName': AdministratorName,
                            'TelephoneNumber': TelephoneNumber,
                            'ClosedTimestamp': ClosedTimestamp,
                            'MostRecentLicenseTimestamp': MostRecentLicenseTimestamp,
                            'LatestUpdateTimestamp': LatestUpdateTimestamp
                            }
                        )


    def handle(self, *args, **options):

        """
        Call the function to import data
        """
        self.import_facility_from_file()