Skip to content

Instantly share code, notes, and snippets.

@elazarl
Created January 30, 2020 15:35
Show Gist options
  • Save elazarl/aaf45f52dbca7e2cc6df33feebb22a91 to your computer and use it in GitHub Desktop.
Save elazarl/aaf45f52dbca7e2cc6df33feebb22a91 to your computer and use it in GitHub Desktop.
Check if a simple transform before applying zstd reduce file size
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <fcntl.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <time.h>
#include <unistd.h>
char buf[1024*1024];
volatile uint64_t sink;
#define USEC_IN_SEC 1000000ULL
#define NSEC_IN_USEC 1000ULL
int read_all(char* buf, size_t buf_size, char *file_name);
int main(int argc, char** argv) {
int narg;
if (argc < 2) {
fputs("too little argumetns\n", stdout);
return 1;
}
uint64_t total_usec = 0;
uint64_t total_bytes = 0;
for (narg = 1; narg < argc; narg++) {
uint64_t i;
int file_size_bytes = read_all(buf, sizeof(buf), argv[narg]);
uint64_t* ptr = (uint64_t*)&buf[0];
struct timespec before, after;
printf("running %d iterations\n", file_size_bytes);
clock_gettime(CLOCK_MONOTONIC, &before);
__sync_synchronize();
for (i=1; i<file_size_bytes/sizeof(*ptr); i++) {
ptr[i] = ptr[i-1]-ptr[i];
}
__sync_synchronize();
clock_gettime(CLOCK_MONOTONIC, &after);
total_bytes += file_size_bytes;
total_usec += ((uint64_t)after.tv_sec - before.tv_sec) * USEC_IN_SEC +
((uint64_t)after.tv_nsec - before.tv_nsec) / 1000;
// make sure compile is obligated to do the above change
for (i=0; i<file_size_bytes/sizeof(*ptr); i++) {
sink = ptr[i];
}
}
const double bytes_in_MB = 1024 * 1024;
printf("%lu bytes (%.03f MB) took %lu usecs\n", total_bytes, total_bytes / bytes_in_MB, total_usec);
double bytes_in_sec = ((double)total_bytes / (double)total_usec) * USEC_IN_SEC;
printf("%.03f MB in sec\n", bytes_in_sec / bytes_in_MB);
}
int read_all(char* buf, size_t buf_size, char *file_name) {
int fd = open(file_name, O_RDONLY);
if (fd < 0) {
perror("error opening file");
exit(2);
}
size_t bytes_read_so_far = 0;
ssize_t bytes_read;
while ((bytes_read = read(fd, buf+bytes_read_so_far, buf_size-bytes_read_so_far)) != 0) {
if (bytes_read < 0) {
perror("error reading file");
close(fd);
exit(1);
}
bytes_read_so_far += bytes_read;
}
close(fd);
return bytes_read_so_far;
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment