Mongo Query String Parser

QueryStringParser.ts

import { toDateOrNull } from '../toDate';

function substituteQuotedCharacters(inner: string) {
    return inner.replace(/\\./g, function(s) {
        switch (s[1]) {
            case 'n':
                return '\n';
            case 't':
                return '\t';
            case 'r':
                return '\r';
        }
        return s[1];
    });
}

function quoteStringIfNeeded(string: string) {
    if (string.match(/[\s\"\\]/)) {
        return '"' + string.replace(/(\n\t\r\\"\\)/g, '\\$1') + '"';
    }
    return string;
}

function unquoteSting(str: string): string {
    // if the string is not quoted, we return the original
    const match = str.match(/^"(.*)"$/);
    if (!match) {
        return str;
    }
    return substituteQuotedCharacters(match[1]);
}

function unquoteRegex(regex: string): string {
    const match = regex.match(/^\/(.*)\/$/);
    if (!match) {
        return regex;
    }
    return substituteQuotedCharacters(match[1]);
}

function toValue(q: string) {
    if (q.match(/^".*"$/)) {
        return unquoteSting(q);
    } else if (q.match(/^\/.*\/$/)) {
        const regexString = unquoteRegex(q);
        return new RegExp(regexString, 'i');
    } else {
        return toObject(q);
    }
}

function toObject(value: string) {
    try {
        return JSON.parse(value);
    } catch (e) {
        // ignore it's not a json object
    }
    if (value.match(/^@\d+/)) {
        try {
            const date = toDateOrNull(value.replace(/^@/, ''));
            if (date) {
                return date;
            }
        } catch (e) {}
    } // const date = new Date(value);
    return value;
}

const op: { [op: string]: string } = {
    '<': '$lt',
    '<=': '$lte',
    '=': '$eq',
    '==': '$eq',
    '!=': '$ne',
    '>=': '$gte',
    '>': '$gt',
};

/**
 * maps comparision operations
 * @param q
 * @returns {any}
 */
function mapOperations(q: string): any {
    if (q.match(/^".*"$/)) {
        q = unquoteSting(q);
    } else if (q.match(/^\/.*\/$/)) {
        const regexString = unquoteRegex(q);
        return new RegExp(regexString, 'i');
    }
    const match = q.match(/^(<=?|==?|!=|>=?)(.+)/);
    if (!match) {
        return toObject(q);
    } else {
        return {
            [op[match[1]]]: toObject(match[2]),
        };
    }
}
// Match key value pair
const fieldRegExp = /^(-?[\w\d_.]+):(.*)/;
const sortRegExp = /^sort:([\w\d_.]+)-(asc|desc)/;
// this regex is a bit complicated
// - the field
// - the second part is either a quoted string, a quoted regex or anything not whitespace
const termSplitRegex = /(-?[\w\d_.]+:(?:\/(?:[^\/\\]+|\\.)*\/|"(?:[^"\\]+|\\.)*"|[^\s]+))|\s+/;

function extractFilters(terms: string[]) {
    // the filter part are all fields that contain a ':'
    const filters = terms.filter(s => s.match(fieldRegExp));

    const andTerms = filters.map(s => {
        const match = s.match(fieldRegExp);
        let field = match[1];
        let value = mapOperations(match[2]);
        // is negation of the terrm
        if (field[0] == '-') {
            field = field.substr(1);
            let negate = '$not';
            if (value === null) {
                negate = '$ne';
            } else if (typeof value !== 'object') {
                negate = '$ne';
            }
            return { [field]: { [negate]: value } };
        } else {
            return { [field]: value };
        }
    });

    // if there is only one search term, we use it as filter
    let filter: any;
    if (andTerms.length == 1) {
        filter = andTerms[0];
    } else if (andTerms.length > 1) {
        filter = { $and: andTerms };
    }
    return filter;
}

function extractSort(sortTerms: string[]): SortTerm {
    if (sortTerms.length == 0) {
        return null;
    }
    let sort: SortTerm = {};

    sortTerms.forEach(term => {
        const match = term.match(sortRegExp);
        if (match) {
            let dir = 1;
            if (match[2] == 'desc') {
                dir = -1;
            }
            sort[match[1]] = dir;
        }
    });
    return sort;
}

export type SortTerm = { [field: string]: number };
export interface ParsedQuery {
    search?: string;
    filter?: any;
    sort?: SortTerm;
}

export function getTerms(query: string): string[] {
    const splitTerms = query.split(termSplitRegex);
    // we now remove any empty string, null or undefined
    return splitTerms.filter(s => s);
}

/**
 * This has been inspired by github query syntax https://help.github.com/articles/search-syntax/
 *
 * - strings are concatenated to one search string.
 * - all search field are combined with AND
 * - the field name is before the `:`
 * - fields of the form `field.subfield:value
 *
 * @param query
 * @returns {any}
 */
export function parseQueryString(query: string): ParsedQuery {
    const terms = getTerms(query);
    // all non fields (which does not contain a :) is joined to the search part of the query
    const search = terms.filter(s => !s.match(fieldRegExp)).join(' ');

    let filter = extractFilters(terms.filter(s => !s.match(sortRegExp)));
    let sort = extractSort(terms.filter(s => s.match(sortRegExp)));

    // construct the result
    const result: any = {};
    // only the non empty fields
    if (search) result.search = search;
    if (filter) result.filter = filter;
    if (sort) result.sort = sort;
    return result;
}

export function setSearchField(query: string, field: string, value: string): string {
    const terms = getTerms(query);
    const regExp = getFieldRegex(field);

    const index = terms.findIndex(term => !!term.match(regExp));
    if (!value) {
        if (index > -1) {
            terms.splice(index, 1);
        }
    } else {
        const newTerm = `${field}:${quoteStringIfNeeded(value)}`;
        if (index < 0) {
            terms.push(newTerm);
        } else {
            terms[index] = newTerm;
        }
    }
    return terms.join(' ');
}

function quoteRegex(field: string) {
    return field.replace(/\./, '\\.');
}

export function getFieldRegex(field: string, excludeNegations = false) {
    if (excludeNegations) {
        return new RegExp(`^${quoteRegex(field)}:`);
    } else {
        return new RegExp(`^-?${quoteRegex(field)}:`);
    }
}

function getFieldValueRaw(query: string, field: string, excludeNegations = false): string {
    const terms = getTerms(query);
    const regExp = getFieldRegex(field, excludeNegations);

    const term = terms.find(t => !!t.match(regExp));
    if (term != null) {
        const match = term.match(fieldRegExp);
        return match[2];
    }
    return undefined;
}

export function getFieldValueString(query: string, field: string): string {
    const value = getFieldValueRaw(query, field, true);
    if (value === undefined) {
        return null;
    }
    return unquoteSting(value);
}

export function getFieldValue(query: string, field: string, excludeNegations = false): string {
    const value = getFieldValueRaw(query, field, excludeNegations);
    if (value === undefined) {
        return undefined;
    }
    return toValue(value);
}

export function containsFieldValue(query: string, field: string, excludeNegations = false): boolean {
    const terms = getTerms(query);
    const regExp = getFieldRegex(field, excludeNegations);
    return terms.findIndex(term => !!term.match(regExp)) > -1;
}

QueryStringParser.utest.ts

import {
    containsFieldValue,
    getFieldValue,
    getFieldValueString,
    parseQueryString,
    setSearchField,
} from './QueryStringParser';
import { assertDeepEqual } from '../../test/assertDeepEqual';
import { assert } from 'chai';

describe('QueryStringParser', function() {
    it('should parse a simple text', function() {
        const result = parseQueryString('test the west');
        assertDeepEqual(result, { search: 'test the west' });
    });
    it('should parse colon list', function() {
        const result = parseQueryString('test  foo:bar the west');
        assertDeepEqual(result, {
            search: 'test the west',
            filter: {
                foo: 'bar',
            },
        });
    });
    it('should parse tow filters into and', function() {
        const result = parseQueryString('test  foo:bar the west other.xxx:filter');
        assertDeepEqual(result, {
            search: 'test the west',
            filter: {
                $and: [{ foo: 'bar' }, { 'other.xxx': 'filter' }],
            },
        });
    });
    it('should parse quoted strings', function() {
        const result = parseQueryString('test the f:>20 foo:bar west foo.bar:"this is a string:2"');
        assertDeepEqual(result, {
            search: 'test the west',
            filter: {
                $and: [{ f: { $gt: 20 } }, { foo: 'bar' }, { 'foo.bar': 'this is a string:2' }],
            },
        });
    });
    it('should parse operations', function() {
        const result = parseQueryString('a:<1 b:<=2 c:=3 d:==4 e:!=5 f:">= 6" g:">7"');
        assertDeepEqual(result, {
            filter: {
                $and: [
                    { a: { $lt: 1 } },
                    { b: { $lte: 2 } },
                    { c: { $eq: 3 } },
                    { d: { $eq: 4 } },
                    { e: { $ne: 5 } },
                    { f: { $gte: 6 } },
                    { g: { $gt: 7 } },
                ],
            },
        });
    });
    it('should parse negations correctly', function() {
        const result = parseQueryString('test  -foo:bar the west -other.xxx:filter -aaa.bbb:>4 -x:!=y');
        assertDeepEqual(result, {
            search: 'test the west',
            filter: {
                $and: [
                    { foo: { $ne: 'bar' } },
                    { 'other.xxx': { $ne: 'filter' } },
                    { 'aaa.bbb': { $not: { $gt: 4 } } },
                    { x: { $not: { $ne: 'y' } } },
                ],
            },
        });
    });
    it('should parse negations of bool and null correctly', function() {
        const result = parseQueryString('-foo:null -bar:true -baz:false');
        assertDeepEqual(result, {
            filter: {
                $and: [{ foo: { $ne: null } }, { bar: { $ne: true } }, { baz: { $ne: false } }],
            },
        });
    });
    it('should parse numbers and booleans', function() {
        const result = parseQueryString('test.bool1:true test.bool2:false negative:-12.4');
        assertDeepEqual(result, {
            filter: {
                $and: [{ 'test.bool1': true }, { 'test.bool2': false }, { negative: -12.4 }],
            },
        });
    });
    it('should parse sort', function() {
        const result = parseQueryString('sort:true field:12 sort:foo.bar-asc sort:bar-desc');
        assertDeepEqual(result, {
            filter: {
                $and: [{ sort: true }, { field: 12 }],
            },
            sort: {
                'foo.bar': 1,
                bar: -1,
            },
        });
    });
    it('should parse regex', function() {
        const result = parseQueryString('field1:/reg ex/ field2:/foo\\/bar/ -field3:/x "\t y/');
        assertDeepEqual(result, {
            filter: {
                $and: [{ field1: /reg ex/i }, { field2: /foo\/bar/i }, { field3: { $not: /x "\t y/i } }],
            },
        });
    });
    it('should parse dates', function() {
        // note zite zone parsing is a bit difficult, so we use Z to make the time neutral
        const result = parseQueryString('created:>@2017 updated:<=@2017-03-28T21:26Z');
        assertDeepEqual(result, {
            filter: {
                $and: [
                    { created: { $gt: new Date('2017-01-01T00:00:00.000Z') } },
                    { updated: { $lte: new Date('2017-03-28T21:26:00.000Z') } },
                ],
            },
        });
    });
    describe('setSearchField', function() {
        it('should add a field', function() {
            const actual = setSearchField('foo:1 bar:"cool string"', 'hello', 'world');
            assert.equal(actual, 'foo:1 bar:"cool string" hello:world');
        });
        it('should replace the field only', function() {
            const actual = setSearchField('foo:1 bar:"cool string" baz:aha', 'bar', 'hello world');
            assert.equal(actual, 'foo:1 bar:"hello world" baz:aha');
        });

        it('should replace the first field only', function() {
            const actual = setSearchField('x:"a b" foo:1 a.b:"a:b" foo:2', 'foo', '3');
            assert.equal(actual, 'x:"a b" foo:3 a.b:"a:b" foo:2');
        });
        it('should remove it when value is null', function() {
            const actual = setSearchField('x:"a b" foo:1 a.b:"a:b" foo:2', 'foo', null);
            assert.equal(actual, 'x:"a b" a.b:"a:b" foo:2');
        });
        it('should deal with `.` in name correctly', function() {
            const actual = setSearchField('axb:1 a.b:2', 'a.b', '3');
            assert.equal(actual, 'axb:1 a.b:3');
        });
        it('should replace -field as if there is no -', function() {
            const actual = setSearchField('x:"a b" -foo:1 a.b:"a:b" foo:2', 'foo', '3');
            assert.equal(actual, 'x:"a b" foo:3 a.b:"a:b" foo:2');
        });
        it('should be cleared by an empty value', function() {
            const actual = setSearchField('x:"a b" -foo:1 a.b:"a:b" foo:2', 'foo', '');
            assert.equal(actual, 'x:"a b" a.b:"a:b" foo:2');
        });
    });
    describe('getFieldValue', function() {
        it('should get string value', function() {
            const actual = getFieldValue('x:"a b" foo:1 a.b:"a:b" foo:2', 'a.b');
            assert.equal(actual, 'a:b');
        });
        it('should get number value', function() {
            const actual = getFieldValue('x:"a b" foo:1 a.b:"a:b" foo:2', 'foo');
            assert.equal(actual as any, 1);
        });

        it('should get undefinded if not in string', function() {
            const actual = getFieldValue('x:"a b" foo:1 a.b:"a:b" foo:2', 'xxx');
            // here we undefined and not null, because null is valid value
            assert.isUndefined(actual);
        });
    });
    describe('getFieldValueString', function() {
        it('should get string value', function() {
            const actual = getFieldValueString('x:"a b" foo:1 a.b:"a:b" foo:2', 'a.b');
            assert.equal(actual, 'a:b');
        });
        it('should get number value', function() {
            const actual = getFieldValueString('x:"a b" foo:1 a.b:"a:b" foo:2', 'foo');
            assert.equal(actual, '1');
        });

        it('should get null if not in string', function() {
            const actual = getFieldValueString('x:"a b" foo:1 a.b:"a:b" foo:2', 'xxx');
            // note in case of a string we use null not undefined
            assert.isNull(actual);
        });

        it('should get null if query is negated', function() {
            const actual = getFieldValueString('x:"a b" -foo:1 a.b:"a:b"', 'foo');
            // note in case of a string we use null not undefined
            assert.isNull(actual);
        });
    });
    describe('containsFieldValue', function() {
        it('should return true if field is contained', function() {
            assert.isTrue(containsFieldValue('x:"a b" foo:1 a.b:"a:b" foo:2', 'a.b'));
        });

        it('should return true if field is not contained', function() {
            assert.isFalse(containsFieldValue('x:"a b" foo:1 axb:"a:b" foo:2', 'a.b'));
        });

        it('should return true if field is negated', function() {
            assert.isTrue(containsFieldValue('x:"a b" foo:1 -axb:"a:b" foo:2', 'axb'));
        });
        it('should return false if field is negated and excludeNegations is set', function() {
            assert.isFalse(containsFieldValue('x:"a b" foo:1 -axb:"a:b" foo:2', 'axb', true));
        });
    });
});