Pulp Download Performance Testing

manytests.sh

#!/bin/bash

rm perf_data.txt

for i in {1..10}
do
   ./perftest.sh
done

parse.py

from datetime import datetime
import json
import sys
import time
import urllib.request


url = sys.argv[1]
url = url[1:-1]  # strip off the double quote characters contained in the url

output_file = sys.argv[2]

while True:
    request = urllib.request.Request(url, headers={'Authorization': 'Basic YWRtaW46YWRtaW4='})
    response_data = urllib.request.urlopen(request).read()
    str_response = response_data.decode('utf-8')
    response = json.loads(str_response)

    if response['state'] != 'completed':
        print('waiting for job to finish')
        time.sleep(15)
    else:
        break

started_at_str = response['started_at']
started_at = datetime.strptime(started_at_str.split('.')[0], '%Y-%m-%dT%H:%M:%S')

finished_at_str = response['finished_at']
finished_at = datetime.strptime(finished_at_str.split('.')[0], '%Y-%m-%dT%H:%M:%S')

runtime = finished_at - started_at

with open(output_file, 'a') as f:
    f.write('{seconds}, '.format(seconds=runtime.seconds))

perftest.sh

#!/bin/bash
source ~/.bashrc.d/alias.bashrc

#read -n 1 -p "Stop the webserver..." mainmenuinput
pkill -9 -f runserver


asyncio=0
num=500

OUTPUTFILE="perf_data.txt"

pstop
sudo rm -rf /var/lib/pulp/artifact/*
pclean
pstart

#read -n 1 -p "Start the webserver..." mainmenuinput
~/.virtualenvs/pulp/bin/python ~/devel/pulp/manage.py runserver &

sleep 5

if [ $asyncio == 1 ]
then
   http POST http://127.0.0.1:8000/api/v3/repositories/ name=example-repo notes:={} scratchpad:={}
   http POST http://127.0.0.1:8000/api/v3/repositories/example-repo/importers/example-asyncio/ name=example-importer download_policy='immediate' repository='http://127.0.0.1:800/api/v3/repositories/example-repo/' feed_url="https://repos.fedorapeople.org/repos/pulp/pulp/demo_repos/file-example/PULP_MANIFEST_$num" sync_mode='mirror'
   URL=$(http POST http://127.0.0.1:8000/api/v3/repositories/example-repo/importers/example-asyncio/example-importer/sync/ | jq '.[]._href')
else
   http POST http://127.0.0.1:8000/api/v3/repositories/ name=example-repo notes:={} scratchpad:={}
   http POST http://127.0.0.1:8000/api/v3/repositories/example-repo/importers/example-futures/ name=example-importer download_policy='immediate' repository='http://127.0.0.1:800/api/v3/repositories/example-repo/' feed_url="https://repos.fedorapeople.org/repos/pulp/pulp/demo_repos/file-example/PULP_MANIFEST_$num" sync_mode='mirror'
   URL=$(http POST http://127.0.0.1:8000/api/v3/repositories/example-repo/importers/example-futures/example-importer/sync/ | jq '.[]._href')
fi

python3 parse.py $URL $OUTPUTFILE