Elasticsearch test setup working on Travis CI

README.md

Searching

After failing to get Elasticsearch::Extensions::Test::Cluster::Cluster, using Rowan Oulton's aproach, to work on Travis CI due to:

Starting 1 Elasticsearch node../usr/local/bin/elasticsearch: line 17: /usr/local/bin/elasticsearch-env: No such file or directory
/usr/local/bin/elasticsearch: line 20: : command not found
/usr/local/bin/elasticsearch: line 24: : command not found
/usr/local/bin/elasticsearch: line 29: exec: : not found
[!!!] Process failed to start (see output above)

We've resorted to using Bonsai as elasticsearch provider in all environments, including test. A simple check can be added to config/initializers/elasticsearch.rb to check the existence if ENV["BONSAI_URL"] and leave the client with the default configuration, thus running locally.

We installed elasticsearch locally with:

brew tap elastic/tap
brew install elastic/tap/elasticsearch-full
brew services start elastic/tap/elasticsearch-full

OurModel.__elasticsearch__.create_index!(force: true)
OurModel.import

elasticsearch.rb

Elasticsearch::Model.client = Elasticsearch::Client.new(url: ENV["BONSAI_URL"])

full_searchable.rb

# frozen_string_literal: true

module FullSearch
  extend ActiveSupport::Concern

  included do
    class << self
      def full_search(text, scope: nil)
        scope ||= all

        return scope if text.blank?

        search = if advanced_search?(text)
          advanced_search(text)
        else
          simple_search(text)
        end

        search.records.merge(scope)
      rescue Elasticsearch::Transport::Transport::Errors::BadRequest
        false
      end

      private

      def advanced_search(text)
        search(query: { query_string: { query: text } })
      end

      def simple_search(text)
        search(query: { simple_query_string: { query: text, default_operator: "AND" } })
      end

      def advanced_search?(text)
        text.match?(/\w+:\w+/)
      end
    end
  end
end

Gemfile

gem "rails", "6.0.3.6"
gem "bonsai-elasticsearch-rails", "~> 7"
gem "elasticsearch-model", github: "elastic/elasticsearch-rails", branch: "master"
gem "elasticsearch-rails", github: "elastic/elasticsearch-rails", branch: "master"

our_model.rb

class OurModel < ApplicationRecord
  include Searchable
  
  # NOTE: apart from the shard and replicas configuration everything here is just to get email searching working somewhat as expected.

  settings(
    index: { number_of_shards: 1, number_of_replicas: production? ? 1 : 0 },
    analysis: {
      analyzer: {
        email_analyzer: { type: "custom", tokenizer: "uax_url_email", filter: %w[lowercase stop] }
      }
    }
  ) do
    mappings dynamic: "true" do
      indexes(
        :email,
        search_analyzer: "email_analyzer",
        analyzer: "standard", # NOTE: required in ES 7.2.0
        fields: {
          partial_email: {
            type: "text",
            analyzer: "email_analyzer"
          }
        }
      )
    end
  end
end

our_model_spec.rb

describe OurModel do
  describe ".full_search", :elasticsearch do
    before_all do
      @tenant1 = create(:tenant, first_name: "Jack", last_name: "Sparrow", email: "pirate@example.com")
      @tenant2 = create(:tenant, first_name: "Elisabeth", last_name: "Swann", email: "governess@example.com")

      @record1 = create(:our_model, number: "123", tenant: @tenant1)
      @record2 = create(:our_model, number: "456", tenant: @tenant2)
      @record3 = create(:our_model, number: "987", tenant: @tenant2, boat_name: "Swann")

      OurModel.refresh_index!
    end

    it "defaults to AND when searching without field" do
      expect(OurModel.full_search("Elisabeth Swann").map(&:number)).to contain_exactly(456, 987)
      expect(OurModel.full_search("Sparrow").map(&:number)).to eq([123])
      expect(OurModel.full_search("Elisabeth Sparrow").map(&:number)).to eq([])
    end

    it "filters as expected on email" do
      expect(OurModel.full_search("pirate@example.com").map(&:number)).to eq([123])
      expect(OurModel.full_search("governess").map(&:number)).to contain_exactly(456, 987)
      expect(OurModel.full_search("example.com").map(&:number)).to contain_exactly(123, 456, 987)
    end

    it "filters on given field" do
      expect(OurModel.full_search("boat_name:Swann").map(&:number)).to eq([987])
    end

    it "filters on given scope" do
      scope = OurModel.where(tenant_id: @tenant2.id)
      expect(OurModel.full_search("example.com", scope: scope).map(&:number)).to contain_exactly(456, 987)
    end

    it "returns all records when given blank query" do
      expect(OurModel.full_search("").map(&:number)).to contain_exactly(123, 456, 987)
    end

    it "returns :bad_query if given bad query" do
      expect(OurModel.full_search(%Q{name:Swann"})).to eq(false)
    end

    it "is re-indexed when record change" do
      @record3.update!(boat_name: "Challenger")
      sleep 1 # Suggestions on how to avoid this is appricated
      expect(OurModel.full_search("Challenger").map(&:number)).to eq([987])
    end
  end

rails_helper.rb

# Parts are inspired by https://bonsai.io/blog/testing-elasticsearch-ruby-gems

Lease.first # NOTE: Init ActiceRecord connection
ES_MODELS = ActiveRecord::Base.descendants.select { |model| model.respond_to?(:__elasticsearch__) }.freeze

RSpec.configure do |config|
  # NOTE: Don't run elasticsearch callbacks unless the spec is tagged with elasticsearch
  config.before do
    unless RSpec.current_example.metadata[:elasticsearch]
      ES_MODELS.each do |model|
        fake_elastic_search = double("__elasticsearch__").as_null_object
        allow_any_instance_of(model).to receive(:__elasticsearch__).and_return(fake_elastic_search) # rubocop:disable RSpec/AnyInstance
      end
    end
  end

  config.before(:all, :elasticsearch) do
    ES_MODELS.each do |model|
      model.__elasticsearch__.create_index!(force: true)
      model.__elasticsearch__.refresh_index!
    end
  end

  config.after(:suite) do
    ES_MODELS.each do |model|
      model.__elasticsearch__.delete_index!
    rescue Elasticsearch::Transport::Transport::Errors::NotFound
      # NOTE: Don't raise "Index does not exist" errors being written to console
    rescue Faraday::ConnectionFailed
      # NOTE: Ignore failure to delete index because of failed connection
    end
  end
end

searchable.rb

module Searchable
  extend ActiveSupport::Concern

  included do
    include Elasticsearch::Model

    index_name [ENV["RAILS_SUB_ENV"].presence || Rails.env, model_name.collection].join("_")

    # NOTE: these are not async as recommended
    after_commit -> { insert_search_index }, on: :create
    after_commit -> { update_search_index }, on: :update
    after_commit -> { delete_search_index }, on: :destroy

    private

    def insert_search_index
      __elasticsearch__.index_document
    end

    def update_search_index
      __elasticsearch__.update_document
    end

    def delete_search_index
      __elasticsearch__.delete_document
    end
  end
end