This is a proof-of-concept for how to turn a Git repository into a tarball binary, which can then be saved in `bytea` or any other binary-compatible SQL column

git_in_sql.js

const http = require('isomorphic-git/http/node');
const git = require('isomorphic-git');
const tar = require('tar-stream');
const memfs = require('memfs');
const stream = require("stream");
const path = require("path");

const REPO_PATH = '/repo';

function streamToBinary (stream) {
  const chunks = []
  return new Promise((resolve, reject) => {
    stream.on('data', chunk => chunks.push(chunk))
    stream.on('error', reject)
    stream.on('end', () => resolve(Buffer.concat(chunks)))
  })
}

async function init_fs() {
    const vol = new memfs.Volume();
    const fs = memfs.createFsFromVolume(vol);
    const dir = REPO_PATH;
    fs.mkdirSync(dir);

    return { fs, vol };
}

async function end_fs({vol}) {
    vol.reset()
}

async function load_repo_from_web(url='https://github.com/mayfer/zpk.git') {
    const { fs, vol } = await init_fs();

    await git.clone({
        fs,
        http,
        dir: REPO_PATH,
        // corsProxy: 'https://cors.isomorphic-git.org',
        url,
    });
    return { fs, vol };
}

function ensureDirectoryExistence(fs, filePath) {
    var dirname = path.dirname(filePath);
    if (fs.existsSync(dirname)) {
        return true;
    }
    ensureDirectoryExistence(fs, dirname);
    fs.mkdirSync(dirname);
}

async function load_repo_from_tar_binary(tar_binary) {
    const { fs, vol } = await init_fs();

    const json_files = {};
    const extract = tar.extract()

    const pack = stream.Readable.from([tar_binary]);

    extract.on('entry', function(header, stream, next) {
        // header is the tar header
        // stream is the content body (might be an empty stream)
        // call next when you are done with this entry
        //json_files[header.name] = await streamToBinary(stream);

        const chunks = [];
        stream.on('data', chunk => chunks.push(chunk))
        stream.on('end', () => {
            console.log(header.name)
            ensureDirectoryExistence(fs, header.name)
            fs.writeFileSync(header.name, Buffer.concat(chunks))
        });

        stream.on('end', function() {
            next() // ready for next entry
        })

        stream.resume() // just auto drain the stream
    })

    extract.on('finish', function() {
      // all entries read;
        // console.log(vol.toJSON())
    })

    pack.pipe(extract)


    return { fs, vol };
}

function walk({fs, dir}) {
    var results = [];
    var list = fs.readdirSync(dir);
    list.forEach(function(file) {
        file = dir + '/' + file;
        var stat = fs.statSync(file);
        if (stat && stat.isDirectory()) {
            /* Recurse into a subdirectory */
            results = results.concat(walk({fs, dir: file}));
        } else {
            /* Is a file */
            results.push(file);
        }
    });
    return results;
}

async function repo_to_tar_binary({fs}) {

    const all_files = walk({fs, dir: REPO_PATH})

    var pack = tar.pack() // pack is a streams2 stream
    all_files.forEach(function(file) {
        pack.entry({ name: file }, fs.readFileSync(file));
    });

    pack.finalize()
     
    // pipe the pack stream somewhere
    // pack.pipe(process.stdout)

    const tar_binary = await streamToBinary(pack);
    return tar_binary;
}

module.exports = {
    walk,
    load_repo_from_web,
    repo_to_tar_binary,
    load_repo_from_tar_binary,
}