creating a large data insert in a django migration

002_auto.py

# Generated by Django 3.1 on 2020-08-19 15:28

from django.db import migrations
from pathlib import Path
import csv

from django.db import transaction


def parse_file(apps, schema_editor):
    State = apps.get_model("states", "State")
    County = apps.get_model("states", "County")
    City = apps.get_model("states", "City")

    us_info = Path("us_cities_states_counties.csv")

    transaction.set_autocommit(False)
    with us_info.open() as info_fp:
        rows = csv.DictReader(info_fp, delimiter="|")
        obj_count = 0
        for row in rows:

            if not row["State full"] or not row["State short"]:
                continue

            state, created = State.objects.get_or_create(
                name=row["State full"], symbol=row["State short"],
            )
            if created:
                print(state.name)
                obj_count += 1

            county, created = County.objects.get_or_create(
                name=row["County"], state=state
            )
            if created:
                obj_count += 1

            city, created = City.objects.get_or_create(name=row["City"], county=county)

            if created:
                obj_count += 1

            if obj_count >= 200:
                transaction.commit()
                obj_count = 0
        transaction.commit()


class Migration(migrations.Migration):
    atomic = False

    dependencies = [
        ("states", "0001_initial"),
    ]

    operations = [
        migrations.RunPython(parse_file,),
    ]