Skip to content

Instantly share code, notes, and snippets.

@DanielGGordon
Last active March 2, 2016 17:43
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
Star You must be signed in to star a gist
Save DanielGGordon/a7da376a34b50ed5e345 to your computer and use it in GitHub Desktop.
Write to a single file in parallel
#include<stdio.h>
#include<stdlib.h>
#include<time.h>
#include<fcntl.h>
#include<unistd.h>
#include<sys/types.h>
#include<sys/stat.h>
#include<errno.h>
struct thread_data {
long int start_offset;
long int blocksize;
int tid;
int iterations;
char *read_buffer;
};
void *doSomeThing(void *thread_arg) {
struct timespec start, finish;
double elapsed;
struct thread_data *my_data;
my_data = (struct thread_data *) thread_arg;
long int offset, br;
int i;
int fdwrite=open("out.txt",O_RDWR);
for(i = 0; i < my_data->iterations; i++)
{
offset = lseek(fdwrite, my_data->start_offset ,SEEK_SET);
br = write(fdwrite, my_data->read_buffer, my_data->blocksize);
my_data->start_offset += my_data->blocksize;
}
close(fdwrite);
}
main(int argc, char *argv[]) {
long int blocksize=atoi(argv[2]);
int iterations_input=atoi(argv[3]);
int iterations;
// RIGHT NOW I AM ONLY USING BLOCKSIZE == 4, WHICH IS 1 GB BLOCKS
if (blocksize == 1) {
blocksize = 1; //1B
iterations = 100000;
}
if (blocksize == 2) {
blocksize = 1024; //1KB
iterations = 10000;
}
if (blocksize == 3) {
blocksize = 1048576; // 1MB
iterations = 1000;
}
if (blocksize == 4) {
blocksize = 1048576 * 1024;
iterations = iterations_input;
}
int i;
int str_length = 2000000;
long int offset = 0;
int seek_counter = blocksize;
//THREADING THREADING ######################################
int NUM_THREADS = atoi(argv[1]);
pthread_t threads[NUM_THREADS]; //creates array of NUM_THREADS
struct thread_data td[NUM_THREADS]; //struct for each thread
i = 0; int br;
iterations = iterations / NUM_THREADS;
int fd=open("read.txt",O_RDONLY);
char *buffer2 = (char *)malloc(blocksize);
br = read(fd, buffer2, blocksize); //buffer2 now stores an array of chars from file, 1 block big
// read() returns the number of bytes read
close(fd);
for (i = 0; i < NUM_THREADS; i++) {
td[i].iterations = iterations;
td[i].blocksize = blocksize;
td[i].tid = i;
td[i].start_offset = offset;
td[i].read_buffer = buffer2;
offset = offset + (iterations * blocksize);
}
int utid; //user made thread id
i = 0;
while (i < NUM_THREADS) {
utid = pthread_create(&threads[i], NULL, &doSomeThing, (void *)&td[i]) ;
i++;
}
pthread_exit(NULL);
}
@DanielGGordon
Copy link
Author

This is called like:

./disk 10 4 100

To use 10 threads to write a 100 gb file in 1 gb blocks.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment