A quick and easy review of timings for lab test loads.

times_reviews.py

from django.core.management.base import BaseCommand

"""
A script to look at timings for loads
"""
import time
from django.utils import timezone
import datetime
from django.conf import settings
import pytds


QUERY = """
SELECT * FROM tQuest.Pathology_Result_View
WHERE date_inserted > @since
"""


def streaming():
    cnt = 0
    ten_days_ago = timezone.now() - datetime.timedelta(days=3)
    start = time.time()
    print(f'starting loading since {ten_days_ago}')
    time_for_first_row = None
    num_of_goes = 0
    length_of_result = 0
    num_of_rows = 0
    array_size = 0
    with pytds.connect(
        settings.TRUST_DB["ip_address"],
        settings.TRUST_DB["database"],
        settings.TRUST_DB["username"],
        settings.TRUST_DB["password"],
        as_dict=True
    ) as conn:
        with conn.cursor() as cur:
            time_for_connection = time.time()
            cur.execute(QUERY, {"since": ten_days_ago})
            while True:
                if not array_size:
                    array_size = cur.arraysize
                result = cur.fetchmany(1000000)
                if not time_for_first_row:
                    time_for_first_row = time.time()
                if result:
                    for row in result:
                        cnt += len(row.keys())
                        num_of_rows += 1
                    num_of_goes += 1
                    if not length_of_result:
                        length_of_result = len(result)
                if not result:
                    break
        end_time = time.time()
        print(f'time for connection {time_for_connection-start}')
        print(f'time for first row {time_for_first_row-time_for_connection}')
        print(f'total time {end_time - start}')
        print(f'num of goes {num_of_goes}')
        print(f'length of result {length_of_result}')
        print(f'array size {array_size}')
        print(f'number of rows {num_of_rows}')


class Command(BaseCommand):
    def handle(self, *args, **options):
        streaming()