GraphQL-Ruby runtime benchmark

runtime_benchmark.rb

# This benchmark aims to test GraphQL-Ruby's runtime performance
# _except_ for parsing.
require "bundler/inline"

gemfile(true) do
  source "https://rubygems.org"
  # Pick a GraphQL version:
  # gem "graphql", "1.13.19"
  gem "graphql", "~>2.0"
  gem "benchmark-ips"
  gem "memory_profiler"
end

require "graphql"
require "benchmark/ips"
require "memory_profiler"

class BenchmarkSchema < GraphQL::Schema
  def self.wrap_proc(value)
    # The value is created at build-time but "hidden" in a proc at runtime
    # to exercise the lazy-resolve runtime feature
    -> { value }
  end

  DATA = 1000.times.map {
    wrap_proc({
        id:             SecureRandom.uuid,
        int1:           SecureRandom.random_number(100000),
        int2:           SecureRandom.random_number(100000),
        string1:        wrap_proc(SecureRandom.base64),
        string2:        SecureRandom.base64,
        boolean1:       SecureRandom.random_number(1) == 0,
        boolean2:       SecureRandom.random_number(1) == 0,
        int_array:      wrap_proc(10.times.map { wrap_proc(SecureRandom.random_number(100000)) } ),
        string_array:   10.times.map { SecureRandom.base64 },
        boolean_array:  10.times.map { SecureRandom.random_number(1) == 0 },
    })
  }

  module Bar
    include GraphQL::Schema::Interface
    field :string_array, [String], null: false
  end

  module Baz
    include GraphQL::Schema::Interface
    implements Bar
    field :int_array, [Integer], null: false
    field :boolean_array, [Boolean], null: false
  end


  class FooType < GraphQL::Schema::Object
    implements Baz
    field :id, ID, null: false
    field :int1, Integer, null: false
    field :int2, Integer, null: false
    field :string1, String, null: false do
      argument :arg1, String, required: false
      argument :arg2, String, required: false
      argument :arg3, String, required: false
      argument :arg4, String, required: false
    end

    field :string2, String, null: false do
      argument :arg1, String, required: false
      argument :arg2, String, required: false
      argument :arg3, String, required: false
      argument :arg4, String, required: false
    end

    field :boolean1, Boolean, null: false do
      argument :arg1, String, required: false
      argument :arg2, String, required: false
      argument :arg3, String, required: false
      argument :arg4, String, required: false
    end
    field :boolean2, Boolean, null: false do
      argument :arg1, String, required: false
      argument :arg2, String, required: false
      argument :arg3, String, required: false
      argument :arg4, String, required: false
    end
  end

  class QueryType < GraphQL::Schema::Object
    description "Query root of the system"
    field :foos, [FooType], null: false, description: "Return a list of Foo objects"
    def foos
      DATA
    end
  end

  query QueryType
  lazy_resolve Proc, :call
end


big_query_document = GraphQL.parse <<-GRAPHQL
  query($skip: Boolean = false) {
    foos {
      id @skip(if: $skip)
      int1
      int2
      string1
      string2
      boolean1
      boolean2
      stringArray
      intArray
      booleanArray
    }
  }
GRAPHQL


puts "Testing with GraphQL v#{GraphQL::VERSION}\n\n"

Benchmark.ips do |x|
  x.config(time: 10)
  x.report("Querying for #{BenchmarkSchema::DATA.size} objects") {
    BenchmarkSchema.execute(document: big_query_document)
  }
  x.report("Introspection") {
    BenchmarkSchema.as_json
  }
end

# For investigating performance:
# result = StackProf.run(mode: :wall, interval: 1) do
#   schema.execute(document: document)
# end
# StackProf::Report.new(result).print_text

puts "\n\n\n"

report = MemoryProfiler.report do
  BenchmarkSchema.execute(document: big_query_document)
end

report.pretty_print

1.13.18:

Querying for 1000 objects
                          3.137  (± 0.0%) i/s -     16.000  in   5.111153s

Total allocated: 23249792 bytes (166490 objects)

2.0.20:

Querying for 1000 objects
                          3.586  (± 0.0%) i/s -     18.000  in   5.032837s

Total allocated: 12736952 bytes (100496 objects)

14% faster, 45% (!?) less memory

2.0.16:

Querying for 1000 objects
                          4.803  (± 0.0%) i/s -     97.000  in  20.205186s


Total allocated: 22355216 bytes (215528 objects)
Total retained:  168 bytes (1 objects)

2.0.21:

Querying for 1000 objects
                          7.543  (± 0.0%) i/s -    151.000  in  20.035807s


Total allocated: 12624648 bytes (100497 objects)
Total retained:  0 bytes (0 objects)

At least the benchmark is getting faster!

I updated the benchmark to include the default introspection query. Looking at 2.0.16 vs. master (will be 2.0.22), Introspection is 60% faster so far:

Testing with GraphQL v2.0.16
Calculating -------------------------------------
Querying for 1000 objects
                          2.934  (± 0.0%) i/s -     30.000  in  10.231301s
       Introspection     44.256  (± 2.3%) i/s -    444.000  in  10.040911s

Testing with master (will be 2.0.22)
Calculating -------------------------------------
Querying for 1000 objects
                          4.703  (± 0.0%) i/s -     47.000  in  10.002045s
       Introspection     70.938  (± 5.6%) i/s -    714.000  in  10.102929s

Comparing 2.0.16 vs 2.0.25, 53% less memory and 79-87% faster:

Testing with GraphQL v2.0.16

Warming up --------------------------------------
Querying for 1000 objects
                         1.000  i/100ms
       Introspection     4.000  i/100ms
Calculating -------------------------------------
Querying for 1000 objects
                          2.787  (± 0.0%) i/s -     28.000  in  10.083777s
       Introspection     44.311  (± 2.3%) i/s -    444.000  in  10.029919s



Total allocated: 22635632 bytes (215528 objects)
Total retained:  168 bytes (1 objects)

Testing with GraphQL v2.0.25

Warming up --------------------------------------
Querying for 1000 objects
                         1.000  i/100ms
       Introspection     7.000  i/100ms
Calculating -------------------------------------
Querying for 1000 objects
                          5.213  (± 0.0%) i/s -     52.000  in  10.009057s
       Introspection     79.516  (± 5.0%) i/s -    798.000  in  10.062841s



Total allocated: 10598880 bytes (75523 objects)
Total retained:  0 bytes (0 objects)

Comparing again, GraphQL-Ruby v2.0.16 vs v2.2.3, on Ruby 3.3.0 without YJIT:

  Calculating -------------------------------------
  Querying for 1000 objects
-                           2.979 (± 0.0%) i/s -     30.000 in  10.077637s
+                           5.968 (± 0.0%) i/s -     60.000 in  10.122830s
-        Introspection     43.965 (± 6.8%) i/s -    440.000 in  10.053077s
+        Introspection     91.018 (± 3.3%) i/s -    909.000 in  10.001223s



- Total allocated: 24239408 bytes (232545 objects)
+ Total allocated: 10749624 bytes (84454 objects)

• 100% and 107% faster
• 55% less memory
• 63% less objects

✨