uploading csv file django using a form

Question

I am working on a school management project and want to upload a CSV file and save it either by overriding the current data or updating it in the database. but it's not being added in the database. Also, I like to add more fields so if possible using dynamic(i.e. by loops) so I don't have to change it later.

models.py

class Student(models.Model):
  registration_number = models.CharField(max_length=200, unique=True)
  firstname = models.CharField(max_length=200)
  surname = models.CharField(max_length=200)
  date_of_birth = models.DateField(default=timezone.now)
  current_class = models.ForeignKey(StudentClass, on_delete=models.SET_NULL, blank=True, null=True)
  date_of_admission = models.DateField(default=timezone.now)
  parent_mobile_number = models.CharField(max_length=15)
  address = models.TextField()

class StudentBulkUpload(models.Model):
  date_uploaded = models.DateTimeField(auto_now=True)
  csv_file = models.FileField(upload_to='students/bulkupload/')

forms.py

class StudentBulkUploadForm(forms.ModelForm):
  class Meta:
    model = StudentBulkUpload
    fields = ("csv_file",)

views.py

def uploadcsv(request):
  if request.method == 'GET':
    form = StudentBulkUploadForm()
    return render(request, 'students/students_upload.html', {'form':form})

  # If not GET method then proceed
  try:
    form = StudentBulkUploadForm(data=request.POST, files=request.FILES)
    if form.is_valid():
      csv_file = form.cleaned_data['csv_file']
    if not csv_file.name.endswith('.csv'):
      messages.error(request, 'File is not CSV type')
      return redirect('students:student-upload')
    # If file is too large
    if csv_file.multiple_chunks():
      messages.error(request, 'Uploaded file is too big (%.2f MB)' %(csv_file.size(1000*1000),))
      return redirect('students:student-upload')
    
    file_data = csv_file.read().decode('utf-8')
    lines = file_data.split('\n')

    # loop over the lines and save them in db. If error, store as string and then display
    for line in lines:
      fields = line.split(',')
      data_dict = {}
      print(data_dict)
      try:
        form = StudentBulkUploadForm(data_dict)
        if form.is_valid():
          form.save()
        else:
          logging.getLogger('error_logger').error(form.errors.as_json())
      except Exception as e:
        logging.getLogger('error_logger').error(form.errors.as_json())
        pass
  except Exception as e:
    logging.getLogger('error_logger').error('Unable to upload file. ' + repr(e))
    messages.error(request, 'Unable to upload file. ' + repr(e))
  return redirect('students:student-upload')

student_upload.html

<form method="POST" enctype="multipart/form-data">
    {% csrf_token %}
    {{ form.as_p }}

    <input type="submit" class="btn btn-success" value="Submit">
</form>

Answer 1

is it necessary that you should pass the data through the form and save it to db. Otherwise you could simply create an object of the model and pass the dictionary to it and save.

I think that you should (in short):

• Upload and save the file first using StudentBulkUpload()
• Get the path for the file and read the contents
• It would be better if you have the same names for model fields and csv columns
• Loop through each line and create a dictionary which contains only a student details at an iteration
• Make an instance of Student() and pass the dictionary to it and save
• For the foreign key, get the object from StudentClass() using get() accordingly with the value that is stored in csv

---

You could save the student details in two ways, I think:

• Create a model object and assign values by reading each line the normal way

new_student = Student()
new_student.registration_number = fields[0]
new_student.firstname = fields[1]
# like so for other fields

new_student.save()

• Create a model object, create a dictionary of key values where keys corresponds to the field names of the model.

# create a dictionary `new_student_details` containing values of a student

new_student = Student()
new_student.__dict__.update(new_student_details)
new_student.save()

---

Create a function to read the csv file and save student details

import csv
def save_new_students_from_csv(file_path):
    # do try catch accordingly
    # open csv file, read lines
    with open(file_path, 'r') as fp:
        students = csv.reader(fp, delimiter=',')
        row = 0
        for student in students:
            if row==0:
                headers = student
                row = row + 1
            else:
                # create a dictionary of student details
                new_student_details = {}
                for i in range(len(headers)):
                    new_student_details[headers[i]] = student[i]

                # for the foreign key field current_class in Student you should get the object first and reassign the value to the key
                new_student_details['current_class'] = StudentClass.objects.get() # get the record according to value which is stored in db and csv file

                # create an instance of Student model
                new_student = Student()
                new_student.__dict__.update(new_student_details)
                new_student.save()
                row = row + 1
        fp.close()

---

Your code should look something like this after:

def uploadcsv(request):
    if request.method == 'GET':
        form = StudentBulkUploadForm()
        return render(request, 'students/students_upload.html', {'form':form})

    # If not GET method then proceed
    try:
        form = StudentBulkUploadForm(data=request.POST, files=request.FILES)
        if form.is_valid():
            csv_file = form.cleaned_data['csv_file']
            if not csv_file.name.endswith('.csv'):
                messages.error(request, 'File is not CSV type')
                return redirect('students:student-upload')
            # If file is too large
            if csv_file.multiple_chunks():
                messages.error(request, 'Uploaded file is too big (%.2f MB)' %(csv_file.size(1000*1000),))
                return redirect('students:student-upload')

            # save and upload file 
            form.save()

            # get the path of the file saved in the server
            file_path = os.path.join(BASE_DIR, form.csv_file.url)

            # a function to read the file contents and save the student details
            save_new_students_from_csv(file_path)
            # do try catch if necessary
                
    except Exception as e:
        logging.getLogger('error_logger').error('Unable to upload file. ' + repr(e))
        messages.error(request, 'Unable to upload file. ' + repr(e))
    return redirect('students:student-upload')

NOTE: The csv file needs to follow proper formatting. You could save Student() anyway you like. Anyhow, the file needs to be uploaded and read. Line by line Student() has to be saved. This is just a structure. Feel free to make necessary changes since I've removed most of your code

Answer 2

In your code, you're doing all the parsing and decoding of the CSV file instead of using already written code. I'd suggest to approach this using the CSV import module for Django. That way you can create a model that automatically takes in CSV data and nicely converts that to a model:

from model import CsvModel

class MyCSvModel(CsvModel):
    student_name = CharField()
    foo = IntegerField()

Then, you'd create a django form as usual that can point to this model and that way you can handle user uploads.

However, if this isn't an option because you already have a certain model that you want to leave untouched (or whatever reason), check this StackOverflow question out that explains how to use the module Pandas to read the csv and transform it into a dataframe.

Hope this helped!