Dynamic eager loading in JSON APIs (my presentation from our local Ruby meetups)

01-intro.md

Dynamic eager loading

• Static: User.includes(:posts => :comments)

• Dynamic: :posts => :comments is a product of user's input

Where did I need this?

• Let's say you're building a JSON API

• You want to expose associations for objects

  {
    "user": {
      "id": 57,
      "nickname": "Jankec",
  ==> "posts": [
        {
          "id": 7,
          "title": "I didn't know Minitest was fun",
  ======> "comments": [
            {"id": 89, ...},
            {"id": 103, ...},
          ]
        }
      ]
    }
  }

• You can expose each association, but user may not need some of them

  • Lots of associations => Enormous response body

  • You can easily get Stack level too deep errors if you're not careful

    class UserMapper < Yaks::Mapper
      has_many :posts
    end
    
    class PostMapper < Yaks::Mapper
      has_one :author, mapper: UserMapper
    end
    
    {
      "user": {
        "posts": [
          {
            "author": {
              "posts": [
                {
                  "author": {
                    ...
                  }
                }
              ]
            }
          }
        ]
      }
    }

• It's easiest if clients can just choose which associations they want

  • /posts?include=author

  • /users/1?include=posts.comments

02-no_eager_loading.rb

require_relative "active_record"

class User < ActiveRecord::Base
  has_many :posts, foreign_key: :author_id
end

class Post < ActiveRecord::Base
  belongs_to :author, class_name: "User"
  has_many :comments
end

class Comment < ActiveRecord::Base
  belongs_to :post
end


# No eager loading

require "yaks"

class UserMapper < Yaks::Mapper
  has_many :posts
end

class PostMapper < Yaks::Mapper
  has_many :comments
end

class CommentMapper < Yaks::Mapper
end


yaks = Yaks.new do
  default_format :json_api
end

yaks.call User.all.to_a

03-static_eager_loading.rb

require_relative "active_record"

class User < ActiveRecord::Base
  has_many :posts, foreign_key: :author_id
end

class Post < ActiveRecord::Base
  belongs_to :author, class_name: "User"
  has_many :comments
end

class Comment < ActiveRecord::Base
  belongs_to :post
end


# Static eager loading

require "yaks"

class UserMapper < Yaks::Mapper
  has_many :posts
end

class PostMapper < Yaks::Mapper
  has_many :comments
end

class CommentMapper < Yaks::Mapper
end


yaks = Yaks.new do
  default_format :json_api
end

yaks.call User.includes(:posts => :comments).to_a

# Problem: All associations will be fetched, regardless of whether they were requested

04-dynamic_eager_loading.rb

require_relative "active_record"

class User < ActiveRecord::Base
  has_many :posts, foreign_key: :author_id
end

class Post < ActiveRecord::Base
  belongs_to :author, class_name: "User"
  has_many :comments
end

class Comment < ActiveRecord::Base
  belongs_to :post
end


# Dynamic eager loading

require "yaks"

class UserMapper < Yaks::Mapper
  has_many :posts
end

class PostMapper < Yaks::Mapper
  has_many :comments
end

class CommentMapper < Yaks::Mapper
end


yaks = Yaks.new do
  default_format :json_api
end

include_query = "posts.comments"

users = include_query.split(",").inject(User) do |users, relationship|
  path = relationship.split(".").map(&:to_sym)
  include_hash = path.reverse.inject { |hash, rel| {rel => hash} }
  users.includes(include_hash)
end

yaks.call users.to_a

05-tactical_eager_loading.rb

require_relative "sequel"

Sequel::Model.plugin :tactical_eager_loading

class User < Sequel::Model
  one_to_many :posts, key: :author_id, reciprocal: :author
end

class Post < Sequel::Model
  many_to_one :author, class: User, reciprocal: :posts
  one_to_many :comments
end

class Comment < Sequel::Model
  many_to_one :post
end


# Dynamic eager loading

require "yaks"

class UserMapper < Yaks::Mapper
  has_many :posts
end

class PostMapper < Yaks::Mapper
  has_many :comments
end

class CommentMapper < Yaks::Mapper
end


yaks = Yaks.new do
  default_format :json_api
end

yaks.call User.all

active_record.rb

require "active_record"
require "logger"

ActiveRecord::Base.establish_connection("postgres:///eager_example")
ActiveRecord::Base.logger = Logger.new(STDOUT).tap do |logger|
  logger.formatter = proc { |severity, datetime, progname, msg| "#{msg}\n" }
end

schema.rb

require "sequel"

DB = Sequel.connect("postgres:///eager_example")

DB.create_table! :users do
  primary_key :id
end

DB.create_table! :posts do
  primary_key :id
  Integer :author_id
end

DB.create_table! :comments do
  primary_key :id
  Integer :post_id
end

users    = DB[:users].returning
posts    = DB[:posts].returning
comments = DB[:comments].returning

3.times.map { users.insert({})[0] }.each do |user|
  3.times.map { posts.insert(author_id: user[:id])[0] }.each do |post|
    3.times.map { comments.insert(post_id: post[:id])[0] }
  end
end

sequel.rb

require "sequel"
require "logger"

DB = Sequel.connect("postgres:///eager_example")
DB.logger = Logger.new(STDOUT).tap do |logger|
  logger.formatter = proc do |severity, datetime, progname, msg|
    "#{msg}\n" if msg.include?("SELECT * FROM")
  end
end