Django : How to import a data from csv to database without using form?

Question

I'm very new to Django, actually very new to coding too. I know it's a silly question, but I have no idea how to make this.

I wanted to import some of data from local csv file and store to database (mine is mysql) without create an upload form (almost of tutorials I've found from google).

I'm very confused of MVC model e.g. where's the part of handling csv should stand? view or model? and also I have to create a function to cut undesired fields from csv. where should I put that code in ?

Here's my model

from __future__ import unicode_literals
import csv, io
from django.conf import settings
from django.db import models

#from django_countries.fields import CountryField

class ASN(models.Model):
    num = models.IntegerField(primary_key=True)
    owner = models.CharField(max_length=50, null=True)
    # Using countryfield to convert from country code to name
    countryCode = models.CharField(max_length=5)
    name = models.CharField(max_length=100, null=True)
    #countryName = CountryField()

    def __str__(self):
        return str(self.owner) + " " + str(self.num) + " " + str(self.countryCode)

class Host(models.Model):
    name = models.CharField(max_length=20)
    id = models.IntegerField(primary_key=True)

    def __str__(self):
        return str(self.id) + " " + str(self.name)

class Peer(models.Model):
    router_ip = models.CharField(max_length=20, primary_key=True)
    bgp_state = models.IntegerField(default=0) 
    as_num = models.ForeignKey('ASN', on_delete=models.CASCADE)
    host_id = models.ForeignKey('Host', on_delete=models.CASCADE)

    def __str__(self):
        return str(self.host_id) + ' ' + str(self.router_ip) + ' ' + str(self.as_num) + ' ' + str(self.bgp_state)

class PeerNeighbor(models.Model):
    neighbor_ip = models.CharField(max_length=20, primary_key=True)
    router_ip = models.ForeignKey('Peer', on_delete=models.CASCADE)

    def __str__(self):
        return str(self.router_ip) + ' ' + str(self.neighbor_ip)

Here's a code to remove unwanted fields (standalone file)

import csv

txt_file_id = r'MR-SG1-BGPPEER.txt'
txt_file_AS = r'show_AS.txt'
csv_file_out = r'file_out.csv'
peer = []
bgp_peer = []
remote_router_id = []
AS_number = []
AS = []
router_ip = []

def main():
    readInput(txt_file_id, txt_file_AS)
    writeOutput(csv_file_out)

def readInput(filename_1, filename_2):
    with open(filename_1, newline='') as csvfile_1:
        spamreader1 = csv.reader(csvfile_1, delimiter=' ', quotechar=" ")
        for row in spamreader1:
            row = ','.join(row)
            row = row.split(',')
            bgp_peer = row[0]
            remote_router_id = row[3]
            bgp_peer = split_list(bgp_peer)
            peer.append(bgp_peer) #store results into list
            router_ip.append(remote_router_id)
        #print(peer)

    with open(filename_2, newline='') as csvfile_2:
        spamreader2 = csv.reader(csvfile_2, delimiter=' ', quotechar=" ")
        for row in spamreader2:
            row = ','.join(row)
            row = row.split(',')
            AS_number = row[3]
            AS.append(AS_number) #store results into list
        #print(AS)
    print(peer, AS)

def writeOutput(filename):
    with open(filename, 'w') as outputFile:
        wr = csv.writer(outputFile, quoting=csv.QUOTE_ALL)
        wr.writerow(zip(router_ip, peer, AS))

def split_list(inputlist):
    string = inputlist.split(".")
    count = 0
    for i in string:
        count+=1
    bgp_peer_ip = string[5:count]
    bgp_peer_ip = '.'.join(bgp_peer_ip)
    return(bgp_peer_ip)   
main()

the second file will give a router_ip, neighbor_ip and asn. Do I have to create a new class in model to keep the data? can I add the data to a particular class instead of create a new one e.g. store router_ip to Class Peer, neighbor_ip to Class PeerNeighbor and store asn to Class ASN.

these are a new class for purpose of keeping data from csv (inside model) but it didn't work.

class dataFromFile(models.Model):
    router_ip = models.CharField(max_length=20, primary_key = True)
    as_num = models.IntegerField(default=0)
    neighbor_ip = models.CharField(max_length=20)
    objects = models.Manager()

def import_db(request):
    f = open('/home/Jobs/Peering_db/file_out.csv')
    for line in f:
        line = line.split(',')
        tmp = dataFromFile.objects.create()
        tmp.router_ip = line[0]
        tmp.neighbor_ip = line[1]
        tmp.as_num = line[2]
        tmp.save()
    f.close()

for update from executing a script, it gave me an errors

(env) bowbth@bowbth:~/django-apps/mysite$ python manage.py shell
Python 3.6.6 (default, Sep 12 2018, 18:26:19) 
[GCC 8.0.1 20180414 (experimental) [trunk revision 259383]] on linux
Type "help", "copyright", "credits" or "license" for more information.
(InteractiveConsole)
>>> exec(open('import_data_csv.py').read())
Traceback (most recent call last):
  File "<console>", line 1, in <module>
  File "<string>", line 16, in <module>

Answer 1

You can create your own script and run with python manage.py shell command:
Your script should be something like this:

#!/usr/bin/env python

"""
    Script to import data from .csv file to Model Database DJango
    To execute this script run: 
                                1) manage.py shell
                                2) exec(open('file_name.py').read())
"""

import csv
from AppName.models import Model1, Model2 

CSV_PATH = '../../your_file_name.csv'      # Csv file path  


with open(CSV_PATH, newline='') as csvfile:
    spamreader = csv.reader(csvfile, delimiter=';', quotechar=';')
    for row in spamreader:
        Model.objects.create(... Attributes here ...)
        # Example -> Book.objects.create(ISBNCode=row[0], title=row[1], author=row[2])

Take a look to my Example in Github
On the other hand, I recommend you take a look to this Answer, here you will find more information about how works with .csv files in Django.

Answer 2

• Create a REST API to accept list of records to be inserted
• Write your cleanup scripts in the View that handles the request
• Use a sample script to read from CSV and form JSON compatible with the API, and send request to your API endpoint with formed JSON as data