comparing the md5 sum of g1kv37 vs hg19

compare.sh

$ curl -s  "ftp://ftp.1000genomes.ebi.ac.uk/vol1/ftp/technical/reference/human_g1k_v37.fasta.gz"  | gunzip -c  | java FastaMD5 > a.txt

$ curl -s "http://hgdownload.cse.ucsc.edu/goldenPath/hg19/bigZips/chromFa.tar.gz" | gunzip -c | tar Oxvf - 2> /dev/null | java FastaMD5 > b.txt

##join
$ join -t '    ' -1 2 -2 2 <(sort -t ' ' -k2,2 a.txt ) <(sort -t '        ' -k2,2 b.txt ) | cut -d ' ' -f 1,2,4 | sort -t '   ' -k3,3

#unjoinable
$ join -t '    ' -1 2 -2 2 -v 1 -v 2  <(sort -t '      ' -k2,2 a.txt ) <(sort -t '        ' -k2,2 b.txt )   | sort -t '       ' -k2,2

FastaMD5.java

import java.io.*;
import java.security.MessageDigest;

public class FastaMD5
{
public static void main(String args[]) throws Exception
       {
       int len=0;
       byte[] buffer = new byte[1];
       MessageDigest complete = null;
	for(;;)
		{
		int c=System.in.read();
		switch(c)
			{
			case -1: case '>':
				{
				if(complete!=null)
					{
					for(byte b:complete.digest())
				       		{
					   	System.out.print(Integer.toString( (b & 0xff ) + 0x100, 16).substring( 1 ));
				       		}
				       	System.out.println("\t"+len);
				       	complete=null;
				       	len=0;
					}
				if(c==-1) return;
				while((c=System.in.read())!=-1 && c!='\n') System.out.print((char)c);
				System.out.print('\t');
				complete=MessageDigest.getInstance("MD5");
				len=0;
				break;
				} 
			case '\n':case ' ':case '\r': break;
			default:
				{
				buffer[0]=(byte)Character.toUpperCase(c);
				complete.update(buffer, 0, 1);
				++len;
				break;
				}
			}
   		}
	}
}

g1kv37_vs_hg19.tsv

unpairable.txt

d89517b400226d3b56e753972a7cad67	chr17_ctg5_hap1	1680828
641e4338fa8d52a5b781bd2a2c08d3c3	chr3	198022430
fa24f81b680df26bcfb6d69b784fbe36	chr4_ctg9_hap1	590426
fe71bc63420d666884f37a3ad79f3317	chr6_apd_hap1	4622290
18c17e1641ef04873b15f40f6c8659a4	chr6_cox_hap2	4795371
2a3c677c426a10e137883ae1ffb8da3f	chr6_dbb_hap3	4610396
9d51d4152174461cd6715c7ddc588dc8	chr6_mann_hap4	4683263
efed415dd8742349cb7aaca054675b9a	chr6_mcf_hap5	4833398
094d037050cad692b57ea12c4fef790f	chr6_qbl_hap6	4611984
3b6d666200e72bcc036bf88a4d7e0749	chr6_ssto_hap7	4928567
d2ed829b8a1628d16cbeee88e88e39eb	chrM	16571
1e86411d73e6f00a10590f976be01623	chrY	59373566


fdfd811849cc2fadebc929bb925902e5	3 dna:chromosome chromosome:GRCh37:3:1:198022430:1	198022430
c68f52674c9fb33aef52dcf399755519	MT gi|251831106|ref|NC_012920.1| Homo sapiens mitochondrion, complete genome	16569
1fa3474750af0948bdf97d5a0ee52e51	Y dna:chromosome chromosome:GRCh37:Y:2649521:59034049:1	59373566