Skip to content

Instantly share code, notes, and snippets.

@sarahhenkens
Created April 1, 2020 00:04
Show Gist options
  • Star 2 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save sarahhenkens/bb0e29afe5b104291e03606326028c62 to your computer and use it in GitHub Desktop.
Save sarahhenkens/bb0e29afe5b104291e03606326028c62 to your computer and use it in GitHub Desktop.
module GraphQL
module Relay
module Cursor
# The encoder/decoder for the cursor used in our smart GraphQL connections
class DefaultEngine
def encode(data)
Base64.strict_encode64(JSON.generate(data))
end
def decode(cursor)
JSON.parse(Base64.strict_decode64(cursor))
end
end
end
end
end
require "graphql/relay/base_connection"
require_relative "./cursor/default_engine"
module GraphQL
module Relay
# Custom connection to support stable cursors, which enabled us to paginate in a stable window
class StableRelationConnection < GraphQL::Relay::BaseConnection
def cursor_from_node(item)
cursor_data = order_fields.map { |f, _result|
if f.is_a?(Sequel::SQL::QualifiedIdentifier)
if f.table == item.class.table_name
item.public_send(f.column)
elsif item.respond_to?(:cursor_value)
item.public_send(:cursor_value, f)
else
raise GraphQL::ExecutionError, "Unable to build cursor from order fields."
end
else
item.public_send(f)
end
}
encode_cursor(cursor_data)
end
def has_next_page
if first
paged_nodes_length >= first && sliced_nodes_count > first
elsif bidirectional? && before
!nodes_with_pk.seek(after: decode_cursor(before)).limit(1).empty?
else
false
end
end
def has_previous_page
if last
paged_nodes_length >= last && sliced_nodes_count > last
elsif bidirectional? && after
!nodes_with_pk.seek(before: decode_cursor(after)).limit(1).empty?
else
false
end
end
def bidirectional?
GraphQL::Relay::ConnectionType.bidirectional_pagination
end
def first
return @first if defined? @first
@first = get_limited_size(:first)
@first
end
def last
return @last if defined? @last
@last = get_limited_size(:last)
@last
end
private
def get_limited_size(argument)
size = get_limited_arg(argument)
size = max_page_size if size && max_page_size && size > max_page_size
size
end
def cursor_engine
@cursor_engine ||= GraphQL::Relay::Cursor::DefaultEngine.new
end
def encode_cursor(data)
cursor_engine.encode(data)
end
def decode_cursor(cursor)
cursor_engine.decode(cursor)
end
# apply first / last limit results
def paged_nodes
return @paged_nodes if defined? @paged_nodes
if first && last
raise ArgumentError, "Cannot use both `first` and `last`"
end
items = sliced_nodes
if first
if relation_limit(items).nil? || relation_limit(items) > first
items = items.limit(first)
end
end
if last
if (relation_limit(items) && last <= relation_limit(items)) || !relation_limit(items)
items = items.limit(last)
end
primary_key = items.model.primary_key
items = items.unfiltered.unlimited.where(
primary_key => items.reverse.select(primary_key)
)
end
if max_page_size && !first && !last
if relation_limit(items).nil? || relation_limit(items) > max_page_size
items = items.limit(max_page_size)
end
end
@paged_nodes = items.all
end
def relation_limit(relation)
relation.opts[:limit]
end
# If a relation contains a `.group` clause, a `.count` will return a Hash.
def relation_count(relation)
count_or_hash = relation.count
count_or_hash.is_a?(Integer) ? count_or_hash : count_or_hash.length
end
# Apply cursors to edges
def sliced_nodes
return @sliced_nodes if defined? @sliced_nodes
@sliced_nodes = nodes_with_pk
@sliced_nodes = @sliced_nodes.seek(after: decode_cursor(after)) if after
@sliced_nodes = @sliced_nodes.seek(before: decode_cursor(before)) if before
@sliced_nodes
end
def nodes_with_pk
return @nodes_with_pk if defined? @nodes_with_pk
@nodes_with_pk = nodes
primary_key = nodes.opts[:model].primary_key
unless order_has_pk?(nodes, primary_key)
@nodes_with_pk = nodes_with_pk.order_append(primary_key).qualify
end
@nodes_with_pk
end
def order_has_pk?(sliced_nodes, pk_name)
order_fields = sliced_nodes.opts[:order] || []
all_order_fields = order_fields.map do |i|
i.is_a?(Sequel::SQL::OrderedExpression) ? i.expression : i
end
all_order_fields.include?(pk_name)
end
def limit_nodes(sliced_nodes, limit)
limit > 0 ? sliced_nodes.limit(limit) : sliced_nodes.where(false)
end
def sliced_nodes_count
return @sliced_nodes_count if defined? @sliced_nodes_count
# If a relation contains a `.group` clause, a `.count` will return a Hash.
@sliced_nodes_count = relation_count(sliced_nodes)
end
def paged_nodes_array
return @paged_nodes_array if defined?(@paged_nodes_array)
@paged_nodes_array = paged_nodes.to_a
end
def paged_nodes_length
paged_nodes_array.length
end
def order_fields
return @order_fields if defined? @order_fields
sliced_nodes.opts[:order].map do |i|
i.is_a?(Sequel::SQL::OrderedExpression) ? i.expression : i
end
end
end
end
end
GraphQL::Relay::BaseConnection.register_connection_implementation(Sequel::Dataset, GraphQL::Relay::StableRelationConnection)
Database::Main.drop_table? :stable_records
Database::Main.create_table :stable_records do
primary_key :id
integer :a, null: false
integer :b, null: false
end
DATA = [
{ id: 1, a: 1, b: 1 },
{ id: 2, a: 1, b: 2 },
{ id: 3, a: 1, b: 3 },
{ id: 4, a: 2, b: 1 },
{ id: 5, a: 2, b: 2 },
{ id: 6, a: 3, b: 1 },
{ id: 7, a: 4, b: 1 },
{ id: 8, a: 4, b: 2 },
{ id: 9, a: 4, b: 3 },
{ id: 10, a: 5, b: 1 }
].freeze
class StableRecord < Sequel::Model(Database::Main[:stable_records])
end
test_input_type = GraphQL::InputObjectType.define do
name "RelayInput"
argument :first, types.Int
argument :last, types.Int
argument :before, types.String
argument :after, types.String
end
GraphQL::Query::Arguments.construct_arguments_class(test_input_type)
class TestCursorEngine
def encode(data)
JSON.generate(data)
end
def decode(cursor)
JSON.parse(cursor)
end
end
describe GraphQL::Relay::StableRelationConnection do
let(:dataset) { StableRecord.order(:id) }
let(:arguments) { {} }
let(:max_page_size) { nil }
subject(:connection) {
args = test_input_type.arguments_class.new(arguments, context: nil, defaults_used: Set.new)
GraphQL::Relay::StableRelationConnection.new(dataset, args, max_page_size: max_page_size)
}
before do
Database::Main[:stable_records].multi_insert(DATA)
allow(connection).to receive(:cursor_engine) { TestCursorEngine.new }
end
describe "#cursor_from_node" do
context "a dataset sorted on its primary key" do
it "only uses the primary key as its cursor value" do
cursor = connection.cursor_from_node(StableRecord[6])
expect(cursor).to eq "[6]"
end
end
context "a dataset sorted on an unstable field" do
let(:dataset) { StableRecord.order(:a, :b) }
it "it appends the primary key to the cursor" do
cursor = connection.cursor_from_node(StableRecord[9])
expect(cursor).to eq "[4,3,9]"
end
end
end
describe "#edge_nodes" do
subject { connection.edge_nodes }
context "With an unsliced dataset" do
context "with no arguments" do
it "Returns the entire dataset" do
expect(subject.length).to eq 10
end
end
context "selecting the first 3 records" do
let(:arguments) { { first: 3 } }
it "Returns the correct records" do
ids = subject.map(&:id)
expect(ids).to eq [1, 2, 3]
end
end
context "selecting the last 4 records" do
let(:arguments) { { last: 4 } }
it "Returns the correct records" do
ids = subject.map(&:id)
expect(ids).to eq [7, 8, 9, 10]
end
end
context "setting both the `first` and `last`" do
let(:arguments) { { first: 2, last: 2 } }
it "raises an error message" do
expect {
subject
}.to raise_error ArgumentError, "Cannot use both `first` and `last`"
end
end
end
context "selecting the first 2 with an `after` cursor" do
let(:arguments) { { after: "[3]", first: 2 } }
it "returns the next 2 records right after the cursor" do
ids = subject.map(&:id)
expect(ids).to eq [4, 5]
end
end
context "selecting the last 4 with an early `after` cursor" do
let(:arguments) { { after: "[2]", last: 4 } }
it "returns the last 4 records in the entire set" do
ids = subject.map(&:id)
expect(ids).to eq [7, 8, 9, 10]
end
end
context "selecting the first 3 with a late `before` cursor" do
let(:arguments) { { before: "[8]", first: 3 } }
it "returns the first 3 records in the entire set" do
ids = subject.map(&:id)
expect(ids).to eq [1, 2, 3]
end
end
context "selecting the last 2 with a `before` cursor" do
let(:arguments) { { before: "[6]", last: 2 } }
it "returns the 2 records right before the cursor record" do
ids = subject.map(&:id)
expect(ids).to eq [4, 5]
end
end
context "slicing the dataset between a `before` and `after` cursor" do
let(:arguments) { { before: "[7]", after: "[3]" } }
it "returns the edges between without including the cursor edges" do
ids = subject.map(&:id)
expect(ids).to eq [4, 5, 6]
end
end
context "with a dataset that is sorted in primary key in desc order" do
let(:dataset) { StableRecord.order(Sequel.desc(:id)) }
context "selecting the first 2 with an `after` cursor" do
let(:arguments) { { after: "[9]", first: 2 } }
it "returns the next 2 edges in the correct DESC order" do
ids = subject.map(&:id)
expect(ids).to eq [8, 7]
end
end
context "selecting the first 5 with an early `before` cursor" do
let(:arguments) { { before: "[7]", first: 5 } }
it "returns the previous 3 edges in the correct DESC order" do
ids = subject.map(&:id)
expect(ids).to eq [10, 9, 8]
end
end
context "selecting the last 3 records in the entire set" do
let(:arguments) { { last: 3 } }
it "returns the last edges in DESC order" do
ids = subject.map(&:id)
expect(ids).to eq [3, 2, 1]
end
end
end
context "with a limit set on the relationship" do
let(:dataset) { StableRecord.limit(3) }
context "selecting the first 4, which is more than the limit" do
let(:arguments) { { first: 4 } }
it "only returns up to the maximum nodes" do
ids = subject.map(&:id)
expect(ids).to eq [1, 2, 3]
end
end
context "selecting the last 4, which is more than the max" do
let(:arguments) { { last: 4 } }
it "only returns up to the maximum nodes" do
ids = subject.map(&:id)
expect(ids).to eq [8, 9, 10]
end
end
end
context "with a max page size set" do
let(:max_page_size) { 5 }
it "it limits the dataset if we do not pass first or last" do
ids = subject.map(&:id)
expect(ids).to eq [1, 2, 3, 4, 5]
end
context "and a lower dataset limit set" do
let(:dataset) { StableRecord.limit(3) }
it "it limits the dataset up to the limit, not the max page size" do
ids = subject.map(&:id)
expect(ids).to eq [1, 2, 3]
end
end
context "and a higher dataset limit set" do
let(:dataset) { StableRecord.limit(7) }
it "it limits the dataset up to the max page size" do
ids = subject.map(&:id)
expect(ids).to eq [1, 2, 3, 4, 5]
end
end
end
end
describe "#has_next_page" do
subject { connection.has_next_page }
context "without any arguments or configurations" do
it { is_expected.to be false }
end
context "when selecting the first 3 records" do
let(:arguments) { { first: 3 } }
it { is_expected.to be true }
end
context "when selecting the first 3 records with the 4th as the `before` cursor" do
let(:arguments) { { first: 3, before: "[4]" } }
it { is_expected.to be false }
end
context "when selecting the first 3 records with the 5th as the `before` cursor" do
let(:arguments) { { first: 3, before: "[5]" } }
it { is_expected.to be true }
end
context "when selecting the last 2 records" do
let(:arguments) { { last: 2 } }
it { is_expected.to be false }
end
context "with bidirectional pagination disabled" do
before { expect(connection).to receive(:bidirectional?) { false } }
context "when selecting the last 2 records before a `before` cursor" do
let(:arguments) { { before: "[6]", last: 2 } }
it { is_expected.to be false }
end
end
context "with bidirectional pagination enabled" do
before { expect(connection).to receive(:bidirectional?) { true } }
context "when selecting the last 2 records before a `before` cursor" do
let(:arguments) { { before: "[6]", last: 2 } }
it { is_expected.to be true }
end
context "when the very last record is our `before` cursor" do
let(:arguments) { { before: "[10]", last: 2 } }
it { is_expected.to be false }
end
end
end
describe "#has_previous_page" do
subject { connection.has_previous_page }
context "without any arguments or configurations" do
it { is_expected.to be false }
end
context "when selecting the first 2 records" do
let(:arguments) { { first: 2 } }
it { is_expected.to be false }
end
context "when selecting the last 4 records" do
let(:arguments) { { last: 4 } }
it { is_expected.to be true }
end
context "with bidirectional pagination disabled" do
before { expect(connection).to receive(:bidirectional?) { false } }
context "when selecting the first 2 records after an `after` cursor" do
let(:arguments) { { after: "[4]", first: 2 } }
it { is_expected.to be false }
end
end
context "with bidirectional pagination enabled" do
before { expect(connection).to receive(:bidirectional?) { true } }
context "when selecting the first 2 records after an `after` cursor" do
let(:arguments) { { after: "[4]", first: 2 } }
it { is_expected.to be true }
end
context "when the very first record is our `after` cursor" do
let(:arguments) { { after: "[1]", first: 2 } }
it { is_expected.to be false }
end
end
end
describe "#first" do
subject { connection.first }
context "without any configuration set" do
it { is_expected.to be nil }
end
context "with `first` set to 5" do
let(:arguments) { { first: 5 } }
it { is_expected.to eq 5 }
end
context "with `max_page_size` set to 10" do
let(:max_page_size) { 10 }
context "and `first` set to 25" do
let(:arguments) { { first: 25 } }
it { is_expected.to eq 10 }
end
context "and `first` set to 3" do
let(:arguments) { { first: 3 } }
it { is_expected.to eq 3 }
end
end
end
describe "#last" do
subject { connection.last }
context "without any configuration set" do
it { is_expected.to be nil }
end
context "with `last` set to 3" do
let(:arguments) { { last: 3 } }
it { is_expected.to eq 3 }
end
context "with `max_page_size` set to 12" do
let(:max_page_size) { 12 }
context "and `last` set to 13" do
let(:arguments) { { last: 13 } }
it { is_expected.to eq 12 }
end
context "and `last` set to 11" do
let(:arguments) { { last: 11 } }
it { is_expected.to eq 11 }
end
end
end
end
@Aryk
Copy link

Aryk commented Apr 18, 2020

You need a #to_sym here

https://gist.github.com/sarahhenkens/bb0e29afe5b104291e03606326028c62#file-stable_relation_connection-rb-L11

Should be:

if f.table.to_sym == item.class.table_name

Users can potentially input a string as the qualified class name. There is no rule or constraint from sequel that qualifiers need to be a symbol.

@Aryk
Copy link

Aryk commented Apr 20, 2020

Also, there is another issue, you cannot handle Sequel.lit order fields, I modified slightly...

      def cursor_from_node(item)
        cursor_data = []
        order_fields.each do |f, _result|
          if f.is_a?(Sequel::SQL::QualifiedIdentifier)
            # @aryk: Original git patch didn't have #to_sym, so I added it.
            if f.table.to_sym == item.class.table_name
              cursor_data << item.public_send(f.column)
            elsif item.respond_to?(:cursor_value)
              cursor_data << item.public_send(:cursor_value, f)
            else
              raise GraphQL::ExecutionError, "Unable to build cursor from order fields."
            end
          elsif f.is_a?(Sequel::SQL::PlaceholderLiteralString)
            f.args.each { |x| cursor_data << item.public_send(x.column) }
          else
            cursor_data << item.public_send(f)
          end
        end

        encode_cursor(cursor_data)
      end

@Aryk
Copy link

Aryk commented May 23, 2020

There also was no condition to handle how specific columns would get treated. If you have a created column and want to modify the accuracy of the cursor to use milliseconds, you would have to add a cursor_value function that would apply to all the fields, and circumvent the logic already in +cursor_from_node+. I added the ability to add column specific cursor values.

  # @aryk - The cursors use equality to find their place in pagination. You either need to chop off milliseconds
  # off of all the timestamp columns, or include the precision in the cursor. I chose to keep the precision and fix the
  # cursor. :) See "stable_relation_connection.rb" to see how it gets used.
  def created_at_cursor_value
    created_at&.iso8601(6)
  end

Here is my cursor_from_node function so far. If you make this into a library, would be great if you can include similar functionality.

      def cursor_from_node(item)
        cursor_data = []

        get_cursor_value = -> (item, column) do
          cursor_method = "#{column}_cursor_value"
          item.public_send(item.respond_to?(cursor_method) ? cursor_method : column)
        end

        order_fields.each do |f, _result|
          if f.is_a?(Sequel::SQL::QualifiedIdentifier)
            # @aryk: Original git patch didn't have #to_sym, so I added it.
            if f.table.to_sym == item.class.table_name
              cursor_data << get_cursor_value.call(item, f.column)
            elsif item.respond_to?(:cursor_value)
              cursor_data << item.public_send(:cursor_value, f)
            elsif item.respond_to?(f.column) # last ditch effort
              cursor_data << get_cursor_value.call(item, f.column)
            elsif item.values.key?(f.column) # last last ditch effort
              cursor_data << item.values[f.column]
            else
              raise GraphQL::ExecutionError, "Unable to build cursor from order fields."
            end
          elsif f.is_a?(Sequel::SQL::PlaceholderLiteralString)
            f.args.each { |x| cursor_data << get_cursor_value.call(item, x.column) }
          else
            cursor_data << get_cursor_value.call(item, f)
          end
        end
        encode_cursor(cursor_data)
      end

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment