bewest/README.markdown

## README.markdown

      
    Raw
  

              README.markdown
            
          
    stream-git

Stream content chunks into unique branch.
What

This is a set of functions providing streaming interfaces to js-git library.
Why

This creates a new unique git branch based on incoming chunks of content.
Example

Log:
*   commit b4a7209b343752b7c3c4e8d2a7bd39cefe0e181e
|\  Merge: 6bfc480 30a8cf9
| | Author: Ben West <bewest@gmail.com>
| | Date:   Tue Dec 17 19:08:52 2013 -0800
| |
| |	Merge branch 'master' of gist.github.com:8008603
| |
| * commit 30a8cf9bc86a49903a4d537e0f1cc84b04362eff
| | Author: Ben West <bewest@users.noreply.github.com>
| | Date:   Tue Dec 17 12:09:35 2013 -0800
| |
* |   commit 6bfc4800ed57abd23cd1be822e159ec05e91e19b
|\ \  Merge: 885de5d ee73988
| | | Author: Ben West <bewest@gmail.com>
| | | Date:   Tue Dec 17 19:08:23 2013 -0800
| | |
| | |	  Merge remote-tracking branch 'origin/incoming/upload/ee7398'
| | |
| * | commit ee73988fd244da7262f894a99ac9cd38204e5585
|  /  Author: MY AUTHOR <git@js>
| |   Date:   Mon Dec 16 17:37:32 2013 -0800
| |
| |	  MY JUSTIFICATION
| |
* | commit 885de5d72e5b250dfef7879a17988f9adacbff77
| | Author: Ben West <bewest@gmail.com>
| | Date:   Tue Dec 17 19:07:55 2013 -0800
| |
| |	remove package.json by hand
| |
* |   commit df4a7c71b5c73055084e35769a3fd3ec9c58cfbc
|\ \  Merge: 73fb9f9 124d4ea
| | | Author: Ben West <bewest@gmail.com>
| | | Date:   Tue Dec 17 19:07:13 2013 -0800
| | |
| | |	  Merge remote-tracking branch 'origin/incoming/upload/124d4e'
| | |
| * | commit 124d4ea0036615edf5f9fb71907b641cb9f1caef
|  /  Author: MY AUTHOR <git@js>
| |   Date:   Mon Dec 16 21:44:53 2013 -0800
| |
| |	  MY JUSTIFICATION
| |
* |   commit 73fb9f9103b706ea8e5273caed2fe0dd7ab00ecc
|\ \  Merge: 73536f8 91c110f
| | | Author: Ben West <bewest@gmail.com>
| | | Date:   Tue Dec 17 19:06:59 2013 -0800
| | |
| | |	  Merge remote-tracking branch 'origin/incoming/upload/91c110'
| | |
| * | commit 91c110fa80d613e914b8bd9da884563f54c6ee1b
|  /  Author: MY AUTHOR <git@js>
| |   Date:   Mon Dec 16 22:16:18 2013 -0800
| |
| |	  MY JUSTIFICATION
| |
* |   commit 73536f870d62ebc7e9ce99d482186047df987c99
|\ \  Merge: d31738b bbe5b0f
| |/  Author: Ben West <bewest@gmail.com>
|/|   Date:   Tue Dec 17 19:05:24 2013 -0800
| |
| |	  Merge branch 'incoming/bbe5b0'
| |
| |	  Conflicts:
| |		stream-git.js
| |
| * commit bbe5b0fedfa04d9ac6f1ca9774b748f58825ecdc
|   Author: MY AUTHOR <git@js>
|   Date:   Tue Dec 17 19:01:47 2013 -0800
|
|	MY JUSTIFICATION
|
*   commit d31738be90a7e4d51960f4f8320e5d874cbe53db
|\  Merge: 2f2e979 9c39dc7
| | Author: Ben West <bewest@gmail.com>
| | Date:   Tue Dec 17 09:12:47 2013 -0800
| |
| |	Merge branch 'incoming/upload/9c39dc'
| |
| * commit 9c39dc7b8d387e4288212edadef0b0a4dc61e02c
|   Author: MY AUTHOR <git@js>
|   Date:   Tue Dec 17 09:08:08 2013 -0800
|
|	MY JUSTIFICATION
|
* commit 2f2e979c3340c383d74a3aab11a31becf49cf5f4
  Author: Ben West <bewest@users.noreply.github.com>
  Date:   Tue Dec 17 09:10:56 2013 -0800

* commit ba60d078fdc05d2ebfa0b2deee7809c50347cd59
  Author: MY AUTHOR <git@js>
  Date:   Tue Dec 17 17:53:26 2013 -0800

      MY JUSTIFICATION

* commit 93fb38e6ff926788002f49a5686d03035f7a4ab6
  Author: MY AUTHOR <git@js>
  Date:   Mon Dec 16 22:16:28 2013 -0800

      MY JUSTIFICATION

* commit 8a56e90c3e0d732e7981f1d9f6eb3c30d760d1fc
  Author: MY AUTHOR <git@js>
  Date:   Mon Dec 16 22:02:48 2013 -0800

      MY JUSTIFICATION

* commit ab9850a41b549ff16fbbbc7605261de6d27cad26
  Author: MY AUTHOR <git@js>
  Date:   Mon Dec 16 21:49:43 2013 -0800

      MY JUSTIFICATION

* commit e87ae9238e5971348415eeecd8fcfadb237f45b5
  Author: MY AUTHOR <git@js>
  Date:   Mon Dec 16 21:43:53 2013 -0800

      MY JUSTIFICATION

* commit 1c4ea460cda81ebac5b2ef8c4a157073b11b8ca0
  Author: MY AUTHOR <git@js>
  Date:   Mon Dec 16 21:39:04 2013 -0800

      MY JUSTIFICATION

* commit 1cbf4301951821dbfc74d073264d1e4571711d56
  Author: MY AUTHOR <git@js>
  Date:   Mon Dec 16 21:35:29 2013 -0800

      MY JUSTIFICATION

* commit 98512f36f8c35185521e2ba102c7a556877be6ce
  Author: MY AUTHOR <git@js>
  Date:   Mon Dec 16 21:32:24 2013 -0800

      MY JUSTIFICATION

* commit 87c203ef63bf5704f39c6feb3e8b4d82e5957a83
  Author: MY AUTHOR <git@js>
  Date:   Mon Dec 16 21:09:56 2013 -0800

      MY JUSTIFICATION

* commit 477e9df3171918545fca332a71413e1923743e75
  Author: MY AUTHOR <git@js>
  Date:   Mon Dec 16 18:17:32 2013 -0800

      MY JUSTIFICATION

* commit d535615b6af634cc987fbfb364d56050971ae0e6
  Author: MY AUTHOR <git@js>
  Date:   Mon Dec 16 16:20:55 2013 -0800

      MY JUSTIFICATION

  
## flurbity.txt
howdy world

## incoming.txt
hello world

## stream-git.js
var platform = require('git-node-platform')
  , jsGit = require('js-git')
  , es = require('event-stream')
  , fsDB = require('git-fs-db')(platform)
  ;

var DB = fsDB
  ;

function logger (msg) {
  return es.through(function (chunk) {
    console.log(msg, chunk);
    this.emit('data', chunk);
  });
}

// Most code here is modeled after:
// * https://github.com/creationix/js-git/blob/master/examples/create.js
// * https://gist.github.com/bewest/8008603
// * https://github.com/substack/stream-handbook#transform
// * https://github.com/dominictarr/event-stream#eventstream

/**
 * Blobs takes incoming objects with keys content and name and commits them to the given repo.
 * It will append these blobs to a tree if given.
 * TODO: rename write-blob or blobWriter
 * @constructor Blobs(repo, [tree]) -> readable-stream
 */
function Blobs (repo, tree) {
  tree = tree || { };
  // Use es.map to save each blob, re-emitting it with the saved hash.
  var stream = es.pipeline(es.map(function (chunk, cb) {
    var content = chunk.content;
    var name = chunk.name;
    repo.saveAs("blob", chunk.content, function (err, hash) {
      if (err) { cb(err); throw err; }
      tree[name] = {mode: 0100644, hash: hash, name: name};
      cb(null, tree[name]);
    });
  }));
  return stream;
}

/**
 * Tree consumes all incoming saved blobs (with hash) and re-emits a
 * single tree ready to be saved.
 * TODO: rename blob-tree or blobTree
 */
function Tree (blobs, tree) {

  var tree = tree || { };
  var tr = es.through(
    function write (chunk) {
      // read all data events emitted from stream, appending to
      // committable tree
      tree[chunk.name] = chunk; delete chunk.name;
    }
  , function end (chunk) {
     // re-emit single committable tree at end.
     this.emit('data', tree);
     this.end( );
  });

  return tr;
}

/**
 * trees consumes incoming trees replete with blobs and hashes, saves
 * and re-emits the result.
 * TODO: take commit hash as argument
 * TODO: rename to write-tree? or previewTree
 * TODO: split into two streams, one emitting committable trees with
 *       commit info added (preview), the second to save and re-emit?
 * * preview-tree
 * * write-tree
 */
function trees (repo, root) {
  root = root || { };
  // create an iterating stream which saves everything
  var stream = es.pipeline(es.map( function (tree, cb) {
    console.log("TREE", tree);
    // write tree
    repo.saveAs('tree', tree, function (err, hash) {
      if (err) { cb(err); }
      // XXX: bad defaults
      var commit = {
        tree: hash
      , author: { name: 'MY AUTHOR', email: 'git@js' }
      , committer: { name: 'MY COMMITTER', email: 'git@js' }
      , message: 'MY JUSTIFICATION'
      };
      if (root.parent) {
        commit.parent = root.parent;
      }
      cb(null, commit);
    });

  }));
  return stream;
}

function Commits (repo, opts) {
  // Use a through stream to transform incoming stream of commit stubs into
  // saved commits.  Incoming stream is full of json objects that are formatted
  // to be commits.
  //
  function writer (chunk) {
    // TODO: fill with opts.
    // TODO: provide some mechanism for connecting future commits to
    // this one (for other, more generic use)
    var tr = this;
    console.log('commiting', chunk);
    // save commit
    repo.saveAs('commit', chunk, function (err, hash) {
      console.log('COMMIT', arguments);
      tr.emit('data', {commit: hash, tree: chunk});
    });
  }
  function end ( ) {
    this.end('end');
  }
  return es.through(writer, end);
}

/**
 * Record commits on a branch.
 */
function UpdateBranch(repo, name) {
  // Use a through stream to transform an incoming stream of commits into
  // commits that can be dereferenced on a branch.
  function writer (chunk) {
    var tr = this;
    console.log(chunk, chunk.commit);
    // Create new branch.
    // Use part of commit hash from the tree to name the branch.
    // TODO: consider adding some kind of timestamp in the name
    var tail = chunk.commit.slice(0, 6);
    var middle = 'incoming'
    var parts = [ middle, name, tail ];
    var name = parts.join('/');
    // create a branch
    repo.setHead(name, function (err) {
      console.log("SET REF", arguments, 'UPDATE TO', chunk.commit);
      // pin branch to point at our commit
      repo.updateHead(chunk.commit, function (err) {
        console.log('updated head', arguments);
        tr.emit('data', chunk);
        tr.end( );
      });
    });
  }
  return es.through(writer);
}

/**
 * Poorly named gitStream reads incoming objects with a `name` and `content` field.
 * It saves incoming objects as as blobs, aggregated into a tree, which is
 * committed to a new uniquely identifying branch.
 */
// TODO: seriously re-think through naming
function gitStream(name) {
  name = platform.fs(name);
  var repo = jsGit(DB(name));
  console.log("repo", repo);
  var commits = UpdateBranch(repo, 'upload');
  function each (data, cb) {
    console.log('committed tree ', data);
    cb(null, data);
  }
  var stream = es.pipeline(logger('FOO'), Blobs(repo), Tree( ), trees(repo), Commits(repo), commits, es.map(each));

  return stream;
}

module.exports = gitStream;
module.exports.gitStream = gitStream;

if (!module.parent) {
  var GIT_REPO = platform.fs('./out/test.git');
  var m = {name: 'incoming.txt', content: 'hello world' }
    , o = {name: 'flurbity.txt', content: 'howdy world' }
    , c = {name: 'incoming-one.txt', content: 'foo\tbar' }
    , k = {name: 'incoming-2013.txt', content: 'qux\tqan' }
    ;
  var incoming = es.readArray([m, o/*, c, k*/]);
  function withResults (err, results) {
    console.log("WITH RESULTS", err, results);
  }
  es.pipeline(incoming, gitStream(GIT_REPO), es.writeArray(withResults));

}
	var platform = require('git-node-platform')
	, jsGit = require('js-git')
	, es = require('event-stream')
	, fsDB = require('git-fs-db')(platform)
	;

	var DB = fsDB
	;

	function logger (msg) {
	return es.through(function (chunk) {
	console.log(msg, chunk);
	this.emit('data', chunk);
	});
	}

	// Most code here is modeled after:
	// * https://github.com/creationix/js-git/blob/master/examples/create.js
	// * https://gist.github.com/bewest/8008603
	// * https://github.com/substack/stream-handbook#transform
	// * https://github.com/dominictarr/event-stream#eventstream

	/**
	* Blobs takes incoming objects with keys content and name and commits them to the given repo.
	* It will append these blobs to a tree if given.
	* TODO: rename write-blob or blobWriter
	* @constructor Blobs(repo, [tree]) -> readable-stream
	*/
	function Blobs (repo, tree) {
	tree = tree \|\| { };
	// Use es.map to save each blob, re-emitting it with the saved hash.
	var stream = es.pipeline(es.map(function (chunk, cb) {
	var content = chunk.content;
	var name = chunk.name;
	repo.saveAs("blob", chunk.content, function (err, hash) {
	if (err) { cb(err); throw err; }
	tree[name] = {mode: 0100644, hash: hash, name: name};
	cb(null, tree[name]);
	});
	}));
	return stream;
	}

	/**
	* Tree consumes all incoming saved blobs (with hash) and re-emits a
	* single tree ready to be saved.
	* TODO: rename blob-tree or blobTree
	*/
	function Tree (blobs, tree) {

	var tree = tree \|\| { };
	var tr = es.through(
	function write (chunk) {
	// read all data events emitted from stream, appending to
	// committable tree
	tree[chunk.name] = chunk; delete chunk.name;
	}
	, function end (chunk) {
	// re-emit single committable tree at end.
	this.emit('data', tree);
	this.end( );
	});

	return tr;
	}

	/**
	* trees consumes incoming trees replete with blobs and hashes, saves
	* and re-emits the result.
	* TODO: take commit hash as argument
	* TODO: rename to write-tree? or previewTree
	* TODO: split into two streams, one emitting committable trees with
	* commit info added (preview), the second to save and re-emit?
	* * preview-tree
	* * write-tree
	*/
	function trees (repo, root) {
	root = root \|\| { };
	// create an iterating stream which saves everything
	var stream = es.pipeline(es.map( function (tree, cb) {
	console.log("TREE", tree);
	// write tree
	repo.saveAs('tree', tree, function (err, hash) {
	if (err) { cb(err); }
	// XXX: bad defaults
	var commit = {
	tree: hash
	, author: { name: 'MY AUTHOR', email: 'git@js' }
	, committer: { name: 'MY COMMITTER', email: 'git@js' }
	, message: 'MY JUSTIFICATION'
	};
	if (root.parent) {
	commit.parent = root.parent;
	}
	cb(null, commit);
	});

	}));
	return stream;
	}

	function Commits (repo, opts) {
	// Use a through stream to transform incoming stream of commit stubs into
	// saved commits. Incoming stream is full of json objects that are formatted
	// to be commits.
	//
	function writer (chunk) {
	// TODO: fill with opts.
	// TODO: provide some mechanism for connecting future commits to
	// this one (for other, more generic use)
	var tr = this;
	console.log('commiting', chunk);
	// save commit
	repo.saveAs('commit', chunk, function (err, hash) {
	console.log('COMMIT', arguments);
	tr.emit('data', {commit: hash, tree: chunk});
	});
	}
	function end ( ) {
	this.end('end');
	}
	return es.through(writer, end);
	}

	/**
	* Record commits on a branch.
	*/
	function UpdateBranch(repo, name) {
	// Use a through stream to transform an incoming stream of commits into
	// commits that can be dereferenced on a branch.
	function writer (chunk) {
	var tr = this;
	console.log(chunk, chunk.commit);
	// Create new branch.
	// Use part of commit hash from the tree to name the branch.
	// TODO: consider adding some kind of timestamp in the name
	var tail = chunk.commit.slice(0, 6);
	var middle = 'incoming'
	var parts = [ middle, name, tail ];
	var name = parts.join('/');
	// create a branch
	repo.setHead(name, function (err) {
	console.log("SET REF", arguments, 'UPDATE TO', chunk.commit);
	// pin branch to point at our commit
	repo.updateHead(chunk.commit, function (err) {
	console.log('updated head', arguments);
	tr.emit('data', chunk);
	tr.end( );
	});
	});
	}
	return es.through(writer);
	}

	/**
	* Poorly named gitStream reads incoming objects with a `name` and `content` field.
	* It saves incoming objects as as blobs, aggregated into a tree, which is
	* committed to a new uniquely identifying branch.
	*/
	// TODO: seriously re-think through naming
	function gitStream(name) {
	name = platform.fs(name);
	var repo = jsGit(DB(name));
	console.log("repo", repo);
	var commits = UpdateBranch(repo, 'upload');
	function each (data, cb) {
	console.log('committed tree ', data);
	cb(null, data);
	}
	var stream = es.pipeline(logger('FOO'), Blobs(repo), Tree( ), trees(repo), Commits(repo), commits, es.map(each));

	return stream;
	}

	module.exports = gitStream;
	module.exports.gitStream = gitStream;

	if (!module.parent) {
	var GIT_REPO = platform.fs('./out/test.git');
	var m = {name: 'incoming.txt', content: 'hello world' }
	, o = {name: 'flurbity.txt', content: 'howdy world' }
	, c = {name: 'incoming-one.txt', content: 'foo\tbar' }
	, k = {name: 'incoming-2013.txt', content: 'qux\tqan' }
	;
	var incoming = es.readArray([m, o/, c, k/]);
	function withResults (err, results) {
	console.log("WITH RESULTS", err, results);
	}
	es.pipeline(incoming, gitStream(GIT_REPO), es.writeArray(withResults));

	}