Iterate over CSV rows using pandas in a faster way-CodePudding

I'm trying to read a CSV file through file upload from html template, and iterating over the rows and creating model object.

views.py

 @login_required 
 def uploadStudents1(request): 
     if request.method == 'POST': 
         uploaded_file = request.FILES['document'] 
         ext = os.path.splitext(uploaded_file.name)[-1].lower() 
         if ext == '.csv': 
             data_file = pd.read_csv( 
                 uploaded_file, parse_dates=['date_of_birth']) 
             data_file.fillna('-', inplace=True) 
             for _, row in data_file.iterrows(): 
                 Sem1Students.objects.create( 
                     fname=row['first_name'], 
                     lname=row['last_name'], 
                     reg_no=row['reg_no'], 
                     gender=row['gender'], 
                     birth_date=row['date_of_birth'], 
                 ) 
             messages.success(request, 'Uploaded student details successfully!') 
             return redirect('/students') 
         else: 
             messages.error(request, "Invalid file type. Please upload again.") 
             return render(request, 'students/upload1.html') 
  
     return render(request, "students/upload/upload1.html")

However this process is really slow, it takes like 5-6 seconds to read and create 74 records.

Is there any better way to do this, i.e make the process faster?

CodePudding user response：

You should use bulk_create because create will take time if you do it inside loop

 @login_required 
 def uploadStudents1(request): 
     if request.method == 'POST': 
         uploaded_file = request.FILES['document'] 
         ext = os.path.splitext(uploaded_file.name)[-1].lower() 
         if ext == '.csv': 
             data_file = pd.read_csv( 
                 uploaded_file, parse_dates=['date_of_birth']) 
             data_file.fillna('-', inplace=True)
             semi_st = [] 
             for _, row in data_file.iterrows(): 
                 semi_st.append(Sem1Students( 
                     fname=row['first_name'], 
                     lname=row['last_name'], 
                     reg_no=row['reg_no'], 
                     gender=row['gender'], 
                     birth_date=row['date_of_birth'], 
                 ))
             Sem1Students.objects.bulk_create(semi_st)
             messages.success(request, 'Uploaded student details successfully!') 
             return redirect('/students') 
         else: 
             messages.error(request, "Invalid file type. Please upload again.") 
             return render(request, 'students/upload1.html') 
  
     return render(request, "students/upload/upload1.html")