Migrate knex migrations to liquibase

data_holder.js

exports.queries = [];

exports.holder = [];

gen_liquibase.py

import os
import itertools

import json
import psycopg2
import re

files = os.listdir('dump')

def gen():
    for fn in files:
        yield (fn, list(ddl_stmts(fn)))
        
def ddl_stmts(fn):
    ignore  = ['begin;', 'commit;', 'select']
    for l in json.load(open("dump/"+fn)):
        q = l['query']
        if type(q) == dict:
            q = l['query']
            if len(q) == 2:
                l = {'query': q['text'], 'params': q['values']}
            else:
                l = {'query': q['text']}
        l["query"] = strip_comments(l["query"])
        if "--" in l["query"]:
            print(l["query"])
            raise Exception("Found --")
        if l["query"].strip().split(" ")[0].lower() not in ignore:
            yield l

def migrations():
    grouped = itertools.groupby(gen(), lambda x: x[0].split(".")[0])
    for migration_name, l in grouped:
        l = list(e[1] for e in l)
        if len(l[0]): 
            if len(l) == 1:
                yield {"migration": migration_name, "up": l[0], "down":[]}
            else:
                yield {"migration": migration_name, "up": l[1], "down":l[0]}

def strip_comments(q):
    r = re.compile("--.*")
    return r.sub("", q)

def dictify(l):
    def gen():
        for i, e in enumerate(l, start=1):
            yield str(i), e

    return dict(gen())

def process_query(l):
    re1 = re.compile('[$](\d+)')
    if len(l) == 2 and len(l["params"]):
        q = l["query"]
        ps = l["params"]
        
        with psycopg2.connect("dbname=some_db_name") as conn:
            with conn.cursor() as cur:
                yield cur.mogrify(re1.sub(r'%(\1)s', q), dictify(ps)).decode("utf-8") 
    else:
        yield l["query"]

def liquibase_migrations():
    d = "liquibase"
    for l in migrations():
        m = l["migration"]
        up = l["up"]
        down = l["down"]
        with open("liquibase/{}.sql".format(m), "w") as f:

            f.write('--liquibase formatted sql\n')
            f.write('--changeset knex_migrations:0 splitStatements:false\n')
            for qd in up:
                for q in process_query(qd):
                    f.write(q)
                f.write(";\n")

            for qd in down:
                f.write("--rollback ")
                
                for q in process_query(qd):
                    if "\n" in q:
                        q = q.replace("\n", " ")
                    f.write(q)
                f.write("\n")



if __name__ == "__main__":
    liquibase_migrations()

knex_changes.patch

diff -r a/node_modules/knex/lib/dialects/postgres/index.js b/node_modules/knex/lib/dialects/postgres/index.js
5c5
< 
---
> var data = require('data_holder');
18a19,60
> function FakeClient() {
>   var cl = require("pg").Client;
>   var pg = Object.create(cl.prototype);
>   cl.apply(pg, arguments);
>   this.pg = pg;
>   console.log("created fake client");
>   //console.log(this.pg);
> }
> 
> FakeClient.prototype = {
>   connect: function (f) {
>     console.log(this.pg.connect);
>     var _this = this;
>     this.pg.connect(function(err, con) {
>       f(err, _this);
>     });
>     console.log("Connected");
>   },
>   on: function() {
>     console.log(arguments);
>   },
>   query: function(query, callback){
>     console.log("Querying");
>     if(arguments.length === 3){
>       data.queries.push({query: arguments[0], params:arguments[1]});
>     } else {
>       data.queries.push({query: arguments[0]});
>     }
>     this.pg.query.apply(this.pg, arguments);
>   }
> }
> 
> function FakeConnection() {
>   console.log("Using fake connection");
> }
> 
> var fakeDriver = {
>   Client: FakeClient
> }
> 
> 
> 
46c88
<     return require('pg');
---
>     return fakeDriver;//require('pg');
70a113
>         console.log("Hrere");
71a115
>         console.log("re");
diff -r a/node_modules/knex/lib/migrate/index.js b/node_modules/knex/lib/migrate/index.js
4c4
< 
---
> var data = require('data_holder');
51c51,54
<         return _this._runBatch(_.difference(all, completed), 'up');
---
>         
>         return _this._runBatch(_.difference(all, completed), 'up').then(function (){
>           fs.writeFileSync("dump.json", JSON.stringify(data.holder));  
>         });
367a371
>       var a;
373a378,382
>           a = []
>           var k = {migration: directory + '/' + name, queries:a};
>           console.log("coming here " + directory + '/' + name);
>           data.queries = a;
>           data.holder.push(k);
379a389
>           fs.writeFileSync("dump/" + name + "." + direction, JSON.stringify(a));

Files

• dump.js - Should be at the toplevel directory of the project
• knex_changes.patch - Should be applied onto the toplevel directory of the project.

Generating the migration files

mkdir liquibase
mkdir dump

./node_modules/.bin/knex migrate:latest
./node_modules/.bin/knex migrate:rollback

python3 read_migrations.py

The liquibase migrations will be found in the directory liquibase