martijnvermaat/.gitignore

## .gitignore
.ipynb_checkpoints
*.pdf
*.fa
[abcde].k1
[abcde].k2
[abcde].k3
[abcde].k4
[abcde].k5
[abcde].k6
[abcde].k7
[abcde].k8
[abcde].k9
[abcde].k10
[abcde].k11
[abcde].k12
[abcde].k13
[abcde].k14
[abcde].k15

## README.md

      
    Raw
  

              README.md
            
          
    Profiling k-mer pairwise comparison

We profile k-mer pairwise comparision for k=[1,12] using the
kMer analysis toolkit and programming library for
k-mer profiles.
See the IPython notebook for more information.

  
## a.count.k1
0.542464971542 26120

## a.count.k10
0.663958072662 42492

## a.count.k11
0.972708940506 91608

## a.count.k12
2.24748587608 288264

## a.count.k13
7.51165699959 1074688

## a.count.k14
27.2333760262 4220276

## a.count.k15
108.990545034 16803304

## a.count.k2
0.541092157364 26120

## a.count.k3
0.547954082489 26120

## a.count.k4
0.578961849213 26120

## a.count.k5
0.578741788864 26144

## a.count.k6
0.586236000061 26196

## a.count.k7
0.567816019058 26360

## a.count.k8
0.570757865906 27144

## a.count.k9
0.593155145645 30204

## b.count.k1
5.48644304276 26120

## b.count.k10
5.7480931282 42528

## b.count.k11
6.13077187538 91608

## b.count.k12
7.46066999435 288280

## b.count.k13
13.2450871468 1074684

## b.count.k14
34.4764480591 4220288

## b.count.k15
116.609779835 16803324

## b.count.k2
5.44484806061 26128

## b.count.k3
5.53405499458 26124

## b.count.k4
5.82787203789 26128

## b.count.k5
5.84015107155 26152

## b.count.k6
5.82632112503 26292

## b.count.k7
5.79324197769 26752

## b.count.k8
5.74274420738 27332

## b.count.k9
5.71768307686 30264

## c.count.k1
54.6297440529 26116

## c.count.k10
57.8999540806 43588

## c.count.k11
58.0587861538 91868

## c.count.k12
60.5846529007 288448

## c.count.k13
72.2292790413 1074680

## c.count.k14
93.4170598984 4220460

## c.count.k15
174.824126005 16803412

## c.count.k2
54.5379779339 26120

## c.count.k3
54.4764060974 26120

## c.count.k4
57.6308250427 26124

## c.count.k5
58.1498789787 26144

## c.count.k6
57.4757890701 26264

## c.count.k7
57.5336329937 26756

## c.count.k8
57.6689498425 28424

## c.count.k9
58.0697989464 32832

## distance.default.k1
0.00376987457275 27948

## distance.default.k10
0.248427867889 121492

## distance.default.k11
1.09074687958 397148

## distance.default.k12
4.51135492325 1499740

## distance.default.k13
17.9062130451 5669460

## distance.default.k14
61.405629158 20096264

## distance.default.k15
199.224109173 63755916

## distance.default.k2
0.00374603271484 27944

## distance.default.k3
0.00378704071045 27944

## distance.default.k4
0.0042028427124 28036

## distance.default.k5
0.00458002090454 28204

## distance.default.k6
0.00510907173157 28544

## distance.default.k7
0.00836801528931 29532

## distance.default.k8
0.0185878276825 34008

## distance.default.k9
0.0714981555939 52820

## distance.scale.k1
0.00386595726013 27948

## distance.scale.k10
0.253719091415 122176

## distance.scale.k11
1.1847550869 397036

## distance.scale.k12
4.87614607811 1499924

## distance.scale.k13
19.3254239559 5670424

## distance.scale.k14
68.5020990372 20095424

## distance.scale.k15
216.661634207 63754052

## distance.scale.k2
0.00380110740662 27936

## distance.scale.k3
0.00382900238037 27936

## distance.scale.k4
0.00426483154297 28044

## distance.scale.k5
0.00455498695374 28208

## distance.scale.k6
0.00516986846924 28472

## distance.scale.k7
0.00827503204346 29584

## distance.scale.k8
0.0180509090424 33936

## distance.scale.k9
0.0715341567993 52776

## distance.scale.smooth.k1
0.00382304191589 27940

## distance.scale.smooth.k10
11.709302187 122136

## distance.scale.smooth.k11
46.8763818741 392168

## distance.scale.smooth.k12
191.472046852 1367092

## distance.scale.smooth.k13
694.941875935 4249648

## distance.scale.smooth.k14
1946.50338984 13000560

## distance.scale.smooth.k15
5011.18195319 42178040

## distance.scale.smooth.k2
0.00398802757263 27940

## distance.scale.smooth.k3
0.00459003448486 27940

## distance.scale.smooth.k4
0.00705599784851 27952

## distance.scale.smooth.k5
0.0167629718781 28212

## distance.scale.smooth.k6
0.0539407730103 28568

## distance.scale.smooth.k7
0.206032991409 29584

## distance.scale.smooth.k8
0.806213140488 33940

## distance.scale.smooth.k9
3.21424412727 52784

## distance.smooth.k1
0.0038058757782 27944

## distance.smooth.k10
11.3999941349 121544

## distance.smooth.k11
44.8743479252 392128

## distance.smooth.k12
209.745757103 1367460

## distance.smooth.k13
775.451279879 4249036

## distance.smooth.k14
2038.04450011 13000368

## distance.smooth.k15
4487.34653401 39471116

## distance.smooth.k2
0.00396203994751 27940

## distance.smooth.k3
0.00455594062805 27944

## distance.smooth.k4
0.00734496116638 28048

## distance.smooth.k5
0.0167670249939 28208

## distance.smooth.k6
0.0530319213867 28560

## distance.smooth.k7
0.208234071732 29528

## distance.smooth.k8
0.775527000427 34012

## distance.smooth.k9
3.10026812553 52828

## profile_count.py
#!/usr/bin/env python

import resource
import sys
import timeit

from k_mer import klib


def index(filename, k):
    with open(filename) as f:
        profile = klib.Profile.from_fasta(f, k)


def profile_index(filename, k):
    repeats = timeit.repeat('index({0}, {1})'.format(repr(filename), k),
                            setup='from __main__ import index',
                            repeat=3, number=1)
    usage_time = min(repeats)
    usage_memory = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss

    return usage_time, usage_memory


if __name__ == '__main__':
    if len(sys.argv) != 3:
        sys.stdout.write('Usage: %s SAMPLE K\n' % sys.argv[0])
        sys.exit(1)
    usage_time, usage_memory = profile_index(sys.argv[1], int(sys.argv[2]))
    print usage_time, usage_memory

## profile_count.sh
#!/bin/bash

for fasta in a.fa b.fa c.fa; do
    # Copy it to local storage
    local_fasta=$(mktemp)
    cp $fasta $local_fasta

    # Read it once, so we have no cache effects
    ./profile_count.py $local_fasta 4 > /dev/null

    for k in {1..15}; do
        ./profile_count.py $local_fasta $k > ${fasta%.fa}.count.k${k}
    done

    rm $local_fasta
done

## profile_distance.py
#!/usr/bin/env python


import resource
import sys
import timeit

from k_mer import ProfileFileType, kdistlib, klib


def distance(filename_a, filename_b, **diff_args):
    with ProfileFileType()(filename_a) as f_a, ProfileFileType()(filename_b) as f_b:
        profile_a = klib.Profile.from_file(f_a)
        profile_b = klib.Profile.from_file(f_b)

        if profile_a.length != profile_b.length:
            raise ValueError('trying to compare profiles for different k')

        dist = kdistlib.ProfileDistance(**diff_args)
        dist.distance(profile_a, profile_b)


def profile_distance(filename_a, filename_b, **diff_args):
    repeats = timeit.repeat('distance({0}, {1}, **{2})'.format(repr(filename_a),
                                                               repr(filename_b),
                                                               repr(diff_args)),
                            setup='from __main__ import distance',
                            repeat=3, number=1)
    usage_time = min(repeats)
    usage_memory = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss

    return usage_time, usage_memory


if __name__ == '__main__':
    if len(sys.argv) != 4:
        sys.stdout.write('Usage: %s PROFILE_LEFT PROFILE_RIGHT MODE\n' % sys.argv[0])
        sys.exit(1)
    if sys.argv[3] not in ('default', 'scale', 'smooth', 'scale.smooth'):
        sys.stdout.write('Not a valid mode: %s\n' % sys.argv[3])
        sys.exit(1)

    if sys.argv[3] == 'default':
        usage_time, usage_memory = profile_distance(sys.argv[1], sys.argv[2])
    elif sys.argv[3] == 'scale':
        usage_time, usage_memory = profile_distance(sys.argv[1], sys.argv[2], do_scale=True)
    elif sys.argv[3] == 'smooth':
        usage_time, usage_memory = profile_distance(sys.argv[1], sys.argv[2], do_smooth=True, threshold=10)
    elif sys.argv[3] == 'scale.smooth':
        usage_time, usage_memory = profile_distance(sys.argv[1], sys.argv[2], do_scale=True, do_smooth=True, threshold=10)

    print usage_time, usage_memory

## profile_distance.sh
#!/bin/bash

# Copy fasta files to local storage to speed things up
local_fasta_d=$(mktemp)
local_fasta_e=$(mktemp)
cp d.fa $local_fasta_d
cp e.fa $local_fasta_e

for k in {1..15}; do
    # Create profiles
    kMer count -k $k d.k${k} $local_fasta_d
    kMer count -k $k e.k${k} $local_fasta_e

    # Copy profiles to local storage
    local_d=$(mktemp)
    local_e=$(mktemp)
    cp d.k${k} $local_d
    cp e.k${k} $local_e

    # Read them once, so we have no cache effects
    kMer info $local_d > /dev/null
    kMer info $local_e > /dev/null

    ./profile_distance.py $local_d $local_e default > distance.default.k${k}
    ./profile_distance.py $local_d $local_e scale > distance.scale.k${k}
    ./profile_distance.py $local_d $local_e smooth > distance.smooth.k${k}
    ./profile_distance.py $local_d $local_e scale.smooth > distance.scale.smooth.k${k}

    rm $local_d $local_e
done

rm $local_fasta_d $local_fasta_e

## profiling.ipynb

      
Display the source blob

    
Display the rendered blob

    
    Raw
  

              profiling.ipynb
            
          
      Sorry, something went wrong. Reload?
      Sorry, we cannot display this file.
      Sorry, this file is invalid so it cannot be displayed.
      
          Viewer requires iframe.
	.ipynb_checkpoints
	*.pdf
	*.fa
	[abcde].k1
	[abcde].k2
	[abcde].k3
	[abcde].k4
	[abcde].k5
	[abcde].k6
	[abcde].k7
	[abcde].k8
	[abcde].k9
	[abcde].k10
	[abcde].k11
	[abcde].k12
	[abcde].k13
	[abcde].k14
	[abcde].k15
	#!/usr/bin/env python

	import resource
	import sys
	import timeit

	from k_mer import klib


	def index(filename, k):
	with open(filename) as f:
	profile = klib.Profile.from_fasta(f, k)


	def profile_index(filename, k):
	repeats = timeit.repeat('index({0}, {1})'.format(repr(filename), k),
	setup='from __main__ import index',
	repeat=3, number=1)
	usage_time = min(repeats)
	usage_memory = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss

	return usage_time, usage_memory


	if __name__ == '__main__':
	if len(sys.argv) != 3:
	sys.stdout.write('Usage: %s SAMPLE K\n' % sys.argv[0])
	sys.exit(1)
	usage_time, usage_memory = profile_index(sys.argv[1], int(sys.argv[2]))
	print usage_time, usage_memory
	#!/bin/bash

	for fasta in a.fa b.fa c.fa; do
	# Copy it to local storage
	local_fasta=$(mktemp)
	cp $fasta $local_fasta

	# Read it once, so we have no cache effects
	./profile_count.py $local_fasta 4 > /dev/null

	for k in {1..15}; do
	./profile_count.py $local_fasta $k > ${fasta%.fa}.count.k${k}
	done

	rm $local_fasta
	done
	#!/bin/bash

	# Copy fasta files to local storage to speed things up
	local_fasta_d=$(mktemp)
	local_fasta_e=$(mktemp)
	cp d.fa $local_fasta_d
	cp e.fa $local_fasta_e

	for k in {1..15}; do
	# Create profiles
	kMer count -k $k d.k${k} $local_fasta_d
	kMer count -k $k e.k${k} $local_fasta_e

	# Copy profiles to local storage
	local_d=$(mktemp)
	local_e=$(mktemp)
	cp d.k${k} $local_d
	cp e.k${k} $local_e

	# Read them once, so we have no cache effects
	kMer info $local_d > /dev/null
	kMer info $local_e > /dev/null

	./profile_distance.py $local_d $local_e default > distance.default.k${k}
	./profile_distance.py $local_d $local_e scale > distance.scale.k${k}
	./profile_distance.py $local_d $local_e smooth > distance.smooth.k${k}
	./profile_distance.py $local_d $local_e scale.smooth > distance.scale.smooth.k${k}

	rm $local_d $local_e
	done

	rm $local_fasta_d $local_fasta_e