Skip to content

Instantly share code, notes, and snippets.

@ronan-mch

ronan-mch/plot.py

Last active September 2, 2016 11:02
Show Gist options
  • Save ronan-mch/e0106f0bbb7bed1ac300c1410c5a390d to your computer and use it in GitHub Desktop.
Save ronan-mch/e0106f0bbb7bed1ac300c1410c5a390d to your computer and use it in GitHub Desktop.
Download ZIP
Solr Stress test
Raw
plot.py
import csv
import pandas
from ggplot import *
import numpy
response_vals = []
with open('report.csv', 'r') as csvfile:
reader = csv.reader(csvfile, delimiter=';')
for row in reader:
response_vals.append(float(row[0]))
series_nums = range(1, len(response_vals)+1)
series = pandas.Series(response_vals)
dataframe = pandas.DataFrame({
'num' : series_nums,
'resp_time' : series
})
chart = ggplot(dataframe, aes(x='num', y='resp_time')) + geom_line()
chart.save('chart.png')
print dataframe.describe()
Raw
processed.csv
We can make this file beautiful and searchable if this error is corrected: No commas found in this CSV file in line 0.
0; /solr/metastore/select?wt=ruby&qt=%2Ftoshokan_document&q=%7B%21raw+f%3Dcluster_id_ss+v%3D%24id%7D&id=163720492&fq=access_ss%3Adtu
0; /solr/metastore/select?wt=ruby&facet.field=format&facet.field=author_facet&facet.field=journal_title_facet&fq=format%3Ajournal&fq=access_ss%3Adtu&qt=%2Ftoshokan&q=issn_ss%3A%2803787788%29&rows=1&spellcheck.q=+A+review+of+definitions+Zero+Energy+Building+and+calculation+methodologies&facet=false&f.author_facet.facet.limit=21&f.journal_title_facet.facet.limit=21&sort=score+desc%2C+pub_date_tsort+desc%2C+journal_vol_tsort+desc%2C+journal_issue_tsort+desc%2C+journal_page_start_tsort+asc%2C+title_sort+asc&defType=lucene&fl=%2A&spellcheck=false
0; /solr/metastore/select?wt=ruby&facet.field=format&facet.field=author_facet&facet.field=journal_title_facet&fq=access_ss%3Adtu&fq=format%3A%28article+OR+book+OR+journal+OR+thesis+OR+other%29&qt=%2Ftoshokan&q=+A+review+of+definitions+Zero+Energy+Building+and+calculation+methodologies&rows=2&spellcheck.q=+A+review+of+definitions+Zero+Energy+Building+and+calculation+methodologies&facet=false&f.author_facet.facet.limit=21&f.journal_title_facet.facet.limit=21&sort=score+desc%2C+pub_date_tsort+desc%2C+journal_vol_tsort+desc%2C+journal_issue_tsort+desc%2C+journal_page_start_tsort+asc%2C+title_sort+asc&start=0&fl=%2A
0; /solr/metastore/select?wt=ruby&facet.field=format&facet.field=author_facet&facet.field=journal_title_facet&fq=access_ss%3Adtu&fq=format%3A%28article+OR+book+OR+journal+OR+thesis+OR+other%29&qt=%2Ftoshokan&q=Fundamentals+of+Tribology&rows=10&spellcheck.q=Fundamentals+of+Tribology&facet=true&f.author_facet.facet.limit=21&f.journal_title_facet.facet.limit=21&sort=score+desc%2C+pub_date_tsort+desc%2C+journal_vol_tsort+desc%2C+journal_issue_tsort+desc%2C+journal_page_start_tsort+asc%2C+title_sort+asc
0; /solr/metastore/select?wt=ruby&facet.field=format&facet.field=author_facet&facet.field=journal_title_facet&fq=cluster_id_ss%3A%28NOT+%2A%29&fq=access_ss%3Adtu&fq=format%3A%28article+OR+book+OR+journal+OR+thesis+OR+other%29&qt=%2Ftoshokan&q=Fundamentals+of+Tribology&rows=0&spellcheck.q=Fundamentals+of+Tribology&facet=false&f.author_facet.facet.limit=21&f.journal_title_facet.facet.limit=21&sort=score+desc%2C+pub_date_tsort+desc%2C+journal_vol_tsort+desc%2C+journal_issue_tsort+desc%2C+journal_page_start_tsort+asc%2C+title_sort+asc&stat=false
0; /solr/metastore/select?wt=ruby&facet.field=format&facet.field=author_facet&facet.field=journal_title_facet&fq=cluster_id_ss%3A%28NOT+%2A%29&fq=access_ss%3Adtu&fq=format%3A%28article+OR+book+OR+journal+OR+thesis+OR+other%29&qt=%2Ftoshokan&q=Fundamentals+of+Tribology&rows=0&spellcheck.q=Fundamentals+of+Tribology&facet=false&f.author_facet.facet.limit=21&f.journal_title_facet.facet.limit=21&sort=score+desc%2C+pub_date_tsort+desc%2C+journal_vol_tsort+desc%2C+journal_issue_tsort+desc%2C+journal_page_start_tsort+asc%2C+title_sort+asc&stat=false
1000; /solr/metastore/toshokan?q=%7B%21raw%20f%3Dcluster_id_ss%20v%3D%24id%7D&id=78190969&fl=subformat_s%2Clanguage_ss&wt=json
0; /solr/metastore/toshokan?q=%7B%21raw%20f%3Dcluster_id_ss%20v%3D%24id%7D&id=78190969&fl=subformat_s%2Clanguage_ss%2Cisolanguage_ss&wt=json
0; /solr/metastore/toshokan?q=%7B%21raw%20f%3Dcluster_id_ss%20v%3D%24id%7D&id=78190154&fl=subformat_s%2Clanguage_ss&wt=json
0; /solr/metastore/toshokan?q=%7B%21raw%20f%3Dcluster_id_ss%20v%3D%24id%7D&id=78190154&fl=subformat_s%2Clanguage_ss%2Cisolanguage_ss&wt=json
Raw
report.csv
We can make this file beautiful and searchable if this error is corrected: No commas found in this CSV file in line 0.
0.047662;/solr/metastore/select?wt=ruby&qt=%2Ftoshokan_document&q=%7B%21raw+f%3Dcluster_id_ss+v%3D%24id%7D&id=163720492&fq=access_ss%3Adtu
0.016258;/solr/metastore/select?wt=ruby&facet.field=format&facet.field=author_facet&facet.field=journal_title_facet&fq=format%3Ajournal&fq=access_ss%3Adtu&qt=%2Ftoshokan&q=issn_ss%3A%2803787788%29&rows=1&spellcheck.q=+A+review+of+definitions+Zero+Energy+Building+and+calculation+methodologies&facet=false&f.author_facet.facet.limit=21&f.journal_title_facet.facet.limit=21&sort=score+desc%2C+pub_date_tsort+desc%2C+journal_vol_tsort+desc%2C+journal_issue_tsort+desc%2C+journal_page_start_tsort+asc%2C+title_sort+asc&defType=lucene&fl=%2A&spellcheck=false
0.01205;/solr/metastore/select?wt=ruby&facet.field=format&facet.field=author_facet&facet.field=journal_title_facet&fq=access_ss%3Adtu&fq=format%3A%28article+OR+book+OR+journal+OR+thesis+OR+other%29&qt=%2Ftoshokan&q=+A+review+of+definitions+Zero+Energy+Building+and+calculation+methodologies&rows=2&spellcheck.q=+A+review+of+definitions+Zero+Energy+Building+and+calculation+methodologies&facet=false&f.author_facet.facet.limit=21&f.journal_title_facet.facet.limit=21&sort=score+desc%2C+pub_date_tsort+desc%2C+journal_vol_tsort+desc%2C+journal_issue_tsort+desc%2C+journal_page_start_tsort+asc%2C+title_sort+asc&start=0&fl=%2A
0.1924;/solr/metastore/select?wt=ruby&facet.field=format&facet.field=author_facet&facet.field=journal_title_facet&fq=access_ss%3Adtu&fq=format%3A%28article+OR+book+OR+journal+OR+thesis+OR+other%29&qt=%2Ftoshokan&q=Fundamentals+of+Tribology&rows=10&spellcheck.q=Fundamentals+of+Tribology&facet=true&f.author_facet.facet.limit=21&f.journal_title_facet.facet.limit=21&sort=score+desc%2C+pub_date_tsort+desc%2C+journal_vol_tsort+desc%2C+journal_issue_tsort+desc%2C+journal_page_start_tsort+asc%2C+title_sort+asc
0.010531;/solr/metastore/select?wt=ruby&facet.field=format&facet.field=author_facet&facet.field=journal_title_facet&fq=cluster_id_ss%3A%28NOT+%2A%29&fq=access_ss%3Adtu&fq=format%3A%28article+OR+book+OR+journal+OR+thesis+OR+other%29&qt=%2Ftoshokan&q=Fundamentals+of+Tribology&rows=0&spellcheck.q=Fundamentals+of+Tribology&facet=false&f.author_facet.facet.limit=21&f.journal_title_facet.facet.limit=21&sort=score+desc%2C+pub_date_tsort+desc%2C+journal_vol_tsort+desc%2C+journal_issue_tsort+desc%2C+journal_page_start_tsort+asc%2C+title_sort+asc&stat=false
0.009981;/solr/metastore/select?wt=ruby&facet.field=format&facet.field=author_facet&facet.field=journal_title_facet&fq=cluster_id_ss%3A%28NOT+%2A%29&fq=access_ss%3Adtu&fq=format%3A%28article+OR+book+OR+journal+OR+thesis+OR+other%29&qt=%2Ftoshokan&q=Fundamentals+of+Tribology&rows=0&spellcheck.q=Fundamentals+of+Tribology&facet=false&f.author_facet.facet.limit=21&f.journal_title_facet.facet.limit=21&sort=score+desc%2C+pub_date_tsort+desc%2C+journal_vol_tsort+desc%2C+journal_issue_tsort+desc%2C+journal_page_start_tsort+asc%2C+title_sort+asc&stat=false
0.016357;/solr/metastore/toshokan?q=%7B%21raw%20f%3Dcluster_id_ss%20v%3D%24id%7D&id=78190969&fl=subformat_s%2Clanguage_ss&wt=json
0.013385;/solr/metastore/toshokan?q=%7B%21raw%20f%3Dcluster_id_ss%20v%3D%24id%7D&id=78190969&fl=subformat_s%2Clanguage_ss%2Cisolanguage_ss&wt=json
0.013899;/solr/metastore/toshokan?q=%7B%21raw%20f%3Dcluster_id_ss%20v%3D%24id%7D&id=78190154&fl=subformat_s%2Clanguage_ss&wt=json
0.013535;/solr/metastore/toshokan?q=%7B%21raw%20f%3Dcluster_id_ss%20v%3D%24id%7D&id=78190154&fl=subformat_s%2Clanguage_ss%2Cisolanguage_ss&wt=json
Raw
results.txt
resp_time
count 72913.000000
mean 0.136932
std 0.457922
min 0.004384
25% 0.011916
50% 0.015154
75% 0.021689
max 31.413615
Raw
stress_test.exs
defmodule StressTest do
use Application
require Logger
@file_path "processed.csv"
@output_file "report.csv"
@base_url "http://metastore_url:port"
def start(_type, _args) do
Logger.info "Starting up..."
end
def read_file(path) do
:inets.start
output_file = File.open!(@output_file, [:append])
File.stream!(path)
|> Enum.each(&test(&1, output_file))
end
def test(line, output) do
[interval, url] = line |> String.trim |> String.split("; ")
interval |> String.to_integer |> Kernel./(2) |> Kernel.round |> :timer.sleep
resp_time = time(fn -> query(url) end)
message = Enum.join([resp_time, url], ";") <> "\n"
IO.write(output, message)
end
def query(query_string) do
url = @base_url <> query_string |> String.to_charlist
case :httpc.request(url) do
{:ok, _} -> :ok
{:error, {reason, _}} ->
Logger.error "Error querying #{url} - #{reason}"
:ok
end
end
def time(func) do
func
|> :timer.tc
|> elem(0)
|> Kernel./(1_000_000)
end
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment