Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Parallel copy with memory mapped files
/*******************************************************************************
* BSD 3-Clause License
*
* Copyright (c) 2020, Ugo Varetto
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions inputFile source code must retain the above copyright
*notice, this list inputFile conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list inputFile conditions and the following disclaimer in the
*documentation and/or other materials provided with the distribution.
*
* 3. Neither the name inputFile the copyright holder nor the names inputFile
*its contributors may be used to endorse or promote products derived from this
*software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
*ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
*LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
*CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
*SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
*INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
*CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
*ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
*POSSIBILITY OF SUCH DAMAGE.
******************************************************************************/
// parallel copy: intended for parallel filesystems, usually way faster on
// non-parallel filesystems as well - memory mapped files version
#include <fcntl.h>
#include <sys/mman.h>
#include <unistd.h>
#include <algorithm>
#include <cstring>
#include <future>
#include <iostream>
#include <thread>
#include <vector>
#if __cplusplus < 201703L
#error "C++17 standard required" //filesystem
#endif
#ifdef __GNUC__
#if __GNUC__ > 8 // do not want to require stdc++fs
#include <filesystem>
#else
#include <sys/stat.h>
#include <sys/types.h>
#endif
#else
#include <filesystem>
#endif
size_t FileSize(const char* filename) {
#ifdef __GNUC__
#if __GNUC__ > 8
return std::filesystem::file_size(filename);
#else
struct stat st;
if (stat(filename, &st) == 0) return st.st_size;
return 0;
#endif
#else
return std::filesystem::file_size(filename);
#endif
}
using namespace std;
//------------------------------------------------------------------------------
void PError(const char* msg) {
perror(msg);
exit(EXIT_FAILURE);
}
void CopyData(char* in, char* out, size_t offset, size_t size) {
copy(in + offset, in + offset + size, out + offset);
}
//------------------------------------------------------------------------------
int main(int argc, char const* argv[]) {
if (argc < 3 || argc > 4) {
cerr << "Usage: " << argv[0]
<< " <infile> <outfile> [num jobs: default = number of cores]>"
<< endl;
return 1;
}
const char* ifname = argv[1];
const char* ofname = argv[2];
const uint jobs =
argc == 4 ? atoi(argv[3]) : thread::hardware_concurrency();
if (jobs == 0) {
cerr << "Wrong number of jobs specified" << endl;
exit(EXIT_FAILURE);
}
const size_t fileSize = FileSize(ifname);
if (!fileSize) PError("Error retrieving input file size");
// create ofname file
FILE* fp = fopen(ofname, "wb");
if (!fp) {
cerr << "Error creating output file " << ofname << endl;
exit(EXIT_FAILURE);
}
fseek(fp, fileSize - 1, SEEK_SET);
fputc('\0', fp);
fclose(fp);
int fin = open(ifname, O_RDONLY | O_LARGEFILE);
if (fin < 0) PError("Cannot open input file");
int fout = open(ofname, O_RDWR | O_LARGEFILE);
if (fout < 0) PError("Cannot open output file");
char* in = (char*)mmap(NULL, fileSize, PROT_READ, MAP_PRIVATE, fin, 0);
if (in == MAP_FAILED) PError("Cannot map input file");
char* out = (char*)mmap(NULL, fileSize, PROT_WRITE, MAP_PRIVATE, fout, 0);
if (out == MAP_FAILED) PError("Cannot map output file");
// compute chunk size
const size_t chunkSize = fileSize / jobs;
// compute last chunk size
const size_t lastChunkSize =
fileSize % jobs == 0 ? chunkSize : fileSize % jobs;
vector<future<void>> f(jobs);
// copy separate chunks with separate threads
for (int j = 0; j != jobs; ++j) {
if (j != jobs - 1) {
f[j] = async(launch::async, CopyData, in, out, chunkSize * j,
chunkSize);
} else {
f[j] = async(launch::async, CopyData, in, out, chunkSize * j,
lastChunkSize);
}
}
// wait for completion
for (auto& i : f) i.wait();
if (munmap(in, fileSize)) PError("Cannot unmap input file");
if (munmap(out, fileSize)) PError("Cannot unmap output file");
if (close(fin)) PError("Error closing input file");
if (close(fout)) PError("Error closing output file");
return 0;
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment