How to manipulate and export file trees.

filetree.rf

/*

How to manipulate and export file trees.

The system module $/dirs contains a number of utilities for creating
and copying directories:

	% reflow doc $/dirs
	Declarations
	
	val Groups func(dir dir, re string) [string:dir]
	    Groups assigns each path in a directory to a group according to the passed-in
	    regular expressio, which must have exactly one regexp group. Paths that do not
	    match the expression are filtered out. Group returns a map that maps each group
	    key to a directory of matched values.
	
	val Make func(map [string:file]) dir
	    Make creates a new dir using the given map of paths to files.
	
	val Pick func(dir dir, pattern string) (file, string)
	    Pick returns the first file in a directory matching a glob pattern. Pick fails
	    if no files match.
	
	val Files func(dir dir) [file]
	    Files returns a sorted (by filename) list of files from a directory.
	
	val Copy func(dir dir, url string) unit
	    Copy copies the directory to an extern location.

*/

val dirs = make("$/dirs")
val image = "ubuntu"

// First, an exec can produce a directory directly:
val dirFromExec = exec(image, cpu := 1) (outdir dir) {"
	echo 1 > {{outdir}}/1
	echo 2 > {{outdir}}/2
"}

// We can turn a directory into a map, and vice-versa (through dirs.Make).
val files = map(dirFromExec)

// We can compose a new tree from both directories and new files.
val newFile = exec(image, cpu := 1) (out file) {"
	echo foobar > {{out}}
"}

// This is the new file tree (types for exposition)
val tree [string:file] = [
	"foobar": newFile,
	// As in Javascript, we can concatenate maps and lists with this 
	// "explosion" syntax.
	...files,
]

val Main = {
	newdir := dirs.Make(tree)
	dirs.Copy(newdir, "s3://grail-marius/testdir")
}

/*

Example run:

% reflow run filetree.rf 
2017/10/27 10:57:33 run name: marius@grailbio.com/69170181
2017/10/27 10:57:57 ec2cluster: launched spot instance i-0e027d054752fb273: m3.medium: 3.6GiB 1 2.4TiB
2017/10/27 10:58:54 -> filetree.dirFromExec 1b2c17bb run    exec ubuntu echo 1 > {{outdir}}/1.echo 2 > {{outdir}}/2
2017/10/27 10:59:08 <- filetree.dirFromExec 1b2c17bb ok     exec 0s 4B
2017/10/27 10:59:08 -> filetree.newFile 8c315f0f run    exec ubuntu echo foobar > {{out}}
2017/10/27 10:59:09 <- filetree.newFile 8c315f0f ok     exec 0s 7B
2017/10/27 10:59:09 -> filetree.Main 093e0702 run  extern s3://grail-marius/testdir/ 11B
2017/10/27 10:59:22 <- filetree.Main 093e0702 ok   extern 2s 0B
2017/10/27 10:59:23 total n=3 time=29s
	ident                n   ncache runtime(m) cpu mem(GiB) disk(GiB) tmp(GiB)
	filetree.Main        1   0                                        
	filetree.newFile     1   0                                        
	filetree.dirFromExec 1   0                                        
	
val<>
% aws s3 ls s3://grail-marius/testdir/
2017-10-27 10:59:23          2 1
2017-10-27 10:59:23          2 2
2017-10-27 10:59:23          7 foobar
% 

*/