Python script for provoking and detecting binary data corruption when using Riak Python Client on Windows 7 64 bit running Python 2.7.5 64 bit with the following Python packages: protobuf==2.4.1 riak==2.0.1 riak-pb==1.4.1.1

main.py

import os
import string
import random
import pdb
import shutil
import json

import riak

NUMBER_OF_TESTFILES = 50
FILE_SIZE = 1000000
TESTFILE_DIR = './testdata'
CORRDATA_DIR = './corruptdata'
OKDATA_DIR   = './okdata'

client = riak.RiakClient(
	host='address_to_riak_cluster',
	http_port=8098,
	protocol='http',
)

if not os.path.isdir(TESTFILE_DIR):
	os.makedirs(TESTFILE_DIR)
if not os.path.isdir(CORRDATA_DIR):
	os.makedirs(CORRDATA_DIR)
if not os.path.isdir(OKDATA_DIR):
	os.makedirs(OKDATA_DIR)


def generate_test_files():

	print "\n### MAKING TESTDATA:"
	for i in range(NUMBER_OF_TESTFILES):
		file_name = ''.join(random.choice(string.ascii_uppercase) for x in range(10))
		file_content = os.urandom(FILE_SIZE)
		open(TESTFILE_DIR + '/' + file_name, 'wb').write(file_content)
		print "    Created testfile: %s of size %s bytes"%(TESTFILE_DIR + '/' + file_name, FILE_SIZE)


def upload_to_riak():
	
	files = os.listdir(TESTFILE_DIR)
	print "\n### UPLOADING TESTDATA TO RIAK:"
	for f in files:
		ro = client.bucket('testdata').get(f)
		if not ro.exists:
			encoded_data = open(TESTFILE_DIR+'/'+f, 'rb').read()
			ro.encoded_data = encoded_data
			ro.content_type = 'application/octet-stream'
			ro.store()
			print "    Uploaded %s"%f
		else:
			print "    File %s already exists, skipping upload"%f


def download_and_verify():
	
	files = os.listdir(TESTFILE_DIR)
	print "\n### DOWNLOADING AND VERIFYING:"
	corruption_log = []
	for f in files:
		ro = client.bucket('testdata').get(f)
		if not ro.exists:
			print "    ERROR! Unexpectedly did not find file %s in riak"%f
			break
		encoded_data_local = open(TESTFILE_DIR+'/'+f, 'rb').read()
		if ro.encoded_data != encoded_data_local:
			print "    ERROR! Local and remote version of file %s do not match."%f
			debug_filename = CORRDATA_DIR+'/'+f
			open(debug_filename, 'wb').write(ro.encoded_data)
			corruption_log.append(dict(key=f, status='CORRUPTED'))
			
		else:
			print "    OK.    Local and remote content of file %s match."%f
			open(OKDATA_DIR+'/'+f+'_OK', 'wb').write(ro.encoded_data)
			corruption_log.append(dict(key=f, status='OK'))

	corrupt_cnt = 0
	for dct in corruption_log:
		if dct['status'] != 'OK':
			corrupt_cnt += 1
	if corrupt_cnt:
		print "\n    %s remote files did not math their local equivalent."%corrupt_cnt
		print "    Corrupted remote file versions saved to %s"%CORRDATA_DIR




if __name__ == '__main__':

	if len(os.listdir(TESTFILE_DIR))==0:
		generate_test_files()

	upload_to_riak()

	download_and_verify()

typical output

### MAKING TESTDATA:
    Created testfile: ./testdata/ZBYOKITYEU of size 1000000 bytes
    Created testfile: ./testdata/WNYBZIMJVJ of size 1000000 bytes
    Created testfile: ./testdata/KLYYLVVPTA of size 1000000 bytes
    Created testfile: ./testdata/LVREUMQZJM of size 1000000 bytes
    Created testfile: ./testdata/TGDOWIVZXN of size 1000000 bytes
    Created testfile: ./testdata/ZOWDTKXHBN of size 1000000 bytes
    Created testfile: ./testdata/BUFEDXLRFG of size 1000000 bytes
    Created testfile: ./testdata/ARUDQHWIKQ of size 1000000 bytes
    Created testfile: ./testdata/NOAYUMPFQW of size 1000000 bytes
    Created testfile: ./testdata/EJRWUBJLAZ of size 1000000 bytes

### UPLOADING TESTDATA TO RIAK:
    Uploaded ARUDQHWIKQ
    Uploaded BUFEDXLRFG
    Uploaded EJRWUBJLAZ
    Uploaded KLYYLVVPTA
    Uploaded LVREUMQZJM
    Uploaded NOAYUMPFQW
    Uploaded TGDOWIVZXN
    Uploaded WNYBZIMJVJ
    Uploaded ZBYOKITYEU
    Uploaded ZOWDTKXHBN

### DOWNLOADING AND VERIFYING:
    OK.    Local and remote content of file ARUDQHWIKQ match.
    ERROR! Local and remote version of file BUFEDXLRFG do not match.
    ERROR! Local and remote version of file EJRWUBJLAZ do not match.
    OK.    Local and remote content of file KLYYLVVPTA match.
    ERROR! Local and remote version of file LVREUMQZJM do not match.
    ERROR! Local and remote version of file NOAYUMPFQW do not match.
    OK.    Local and remote content of file TGDOWIVZXN match.
    OK.    Local and remote content of file WNYBZIMJVJ match.
    OK.    Local and remote content of file ZBYOKITYEU match.
    ERROR! Local and remote version of file ZOWDTKXHBN do not match.

    5 remote files did not math their local equivalent.
    Corrupted remote file versions saved to ./corruptdata