Fauna Ngram search example

fauna-ngram-search.md

WARNING! This has only been tested on collections with a few k items. Don't know how this performs on "large" collections or large chunks of text.

Create an index on one or multiple values

q.CreateIndex({
    name: "Foos_ngram",
    active: true,
    source: {
        class: q.Collection("Foos"),
        fields: {
            ngram: q.Query(
                q.Lambda(
                    "instance",
                    q.NGram(
                        q.LowerCase(
                            q.Concat(
                                [
                                    q.Select(
                                        ["data", "email"],
                                        q.Var("instance")
                                    ),
                                    q.Select(
                                        ["data", "firstName"],
                                        q.Var("instance"),
                                        ""
                                    ),
                                    q.Select(
                                        ["data", "lastName"],
                                        q.Var("instance"),
                                        ""
                                    ),
                                ],
                                ""
                            )
                        ),
                        2,
                        3
                    )
                )
            ),
        },
    },
    terms: [{ binding: "ngram" }],
})

Search example

const searchString = "Foo Bar"

if (searchString.length < 2)
    throw new Error("Search needs at least two characters")

const words = searchString.trim().split(/\s/g)

const searchStrings = words
    .map(word => {
        const items = word.match(/.{1,3}/g)
        const length = items.length
        if (items.length > 1 && items[length - 1].length === 1) {
            const last = items[length - 1]
            const beforeLast = items[length - 2]
            items[length - 2] = beforeLast.slice(0, 2)
            items[length - 1] = beforeLast.slice(2, 3) + last
            beforeLast
        }
        return items
    })
    .flat()
    .filter(item => item.length > 1)
 
const matchers = searchStrings.map(part => q.Match(q.Index("Foos_ngram"), q.LowerCase(part)))

const query = q.Intersection(...matchers)