Skip to content

Instantly share code, notes, and snippets.

@croepha

croepha/merge_dir.cpp

Created August 24, 2018 23:13
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save croepha/2ce83cf5b5aeee70cba1dcf8e1b8e902 to your computer and use it in GitHub Desktop.
Save croepha/2ce83cf5b5aeee70cba1dcf8e1b8e902 to your computer and use it in GitHub Desktop.
Download ZIP
filesystem filtering tools
Raw
merge_dir.cpp
// cro_build:release,noasan,raw, -static-libstdc++
#define _XOPEN_SOURCE 700
#define _POSIX_C_SOURCE 200809L
#include <fcntl.h>
#include <sys/stat.h>
#include <unistd.h>
#include <sys/types.h>
#include <dirent.h>
#include <errno.h>
#include <signal.h>
#include <attr/xattr.h>
#include <limits.h>
#include <stdlib.h>
#include "cro_lib/cro_buffer.h"
#define _LOGNAME "merge_dir.cpp"
typedef StringWithLength S;
int progress = 0;
bool need_line = false;
bool do_dir(S src_root,S dest_root, S path) {
{
CroBuffer<2048,false> src_path( src_root, '/', path);
CroBuffer<2048,false> dest_path(dest_root, '/', path);
struct stat dest_stat;
auto r2 = lstat(dest_path.internal_buffer, &dest_stat);
if (r2 == -1) {
assert(errno == ENOENT);
auto r5 = rename(src_path.internal_buffer, dest_path.internal_buffer);
assert(r5 ==0);
return true;
} else if ((dest_stat.st_mode & S_IFMT) != S_IFDIR) {
return false;
}
}
DIR*d;
{
CroBuffer<2048,false> full_path(src_root,'/',path);
d = opendir(full_path.internal_buffer);
assert(d);
}
int source_directory_members = 0;
for(;;) {
dirent de;
{
dirent*der;
auto r1 = readdir_r(d, &de, &der);
assert(r1==0);
if (!der) break;
assert(der == &de);
}
progress++;
if (progress>100) {
putchar('.');
fflush(stdout);
progress = 0;
}
if (strcmp(de.d_name, ".")==0) {
continue;
} else if (strcmp(de.d_name, "..")==0){
continue;
} else if (strcmp(de.d_name, ".DS_Store")==0){
CroBuffer<2048,false> src_path(src_root, '/', path, '/', de.d_name);
auto r2 = unlink(src_path.internal_buffer);
assert(r2 == 0);
continue;
}
source_directory_members++;
if (de.d_type == DT_REG) {
bool dest_exists = false;
bool dest_differs = false;
CroBuffer<2048,false> dest_path(dest_root, '/', path, '/', de.d_name);
CroBuffer<2048,false> src_path(src_root, '/', path, '/', de.d_name);
struct stat dest_stat;
auto r2 = lstat(dest_path.internal_buffer, &dest_stat);
if (r2 == -1) {
assert(errno == ENOENT);
} else {
dest_exists = true;
if (de.d_type == DT_REG) {
if ((dest_stat.st_mode & S_IFMT) == S_IFREG) {
struct stat src_stat;
auto r3 = lstat(src_path.internal_buffer, &src_stat);
assert(r3==0);
if (src_stat.st_size == dest_stat.st_size) {
u32 src_crc;
auto r1 = getxattr(src_path.internal_buffer, "user.cro_cksum", &src_crc, sizeof src_crc);
assert(r1 == sizeof src_crc);
u32 dest_crc;
auto r6 = getxattr(dest_path.internal_buffer, "user.cro_cksum", &dest_crc, sizeof dest_crc);
assert(r6 == sizeof dest_crc);
if (src_crc != dest_crc) {
dest_differs = true;
}
} else {
dest_differs = true;
}
} else if ((dest_stat.st_mode & S_IFMT) == S_IFDIR) {
dest_differs = true;
}
} else if (de.d_type == DT_DIR) {
if ((dest_stat.st_mode & S_IFMT) == S_IFREG) {
dest_differs = true;
} else if ((dest_stat.st_mode & S_IFMT) == S_IFDIR) {
}
}
}
if (dest_exists) {
if (!dest_differs) {
auto r4 = unlink(src_path.internal_buffer);
assert(r4 == 0);
source_directory_members--;
}
} else {
auto r5 = rename(src_path.internal_buffer, dest_path.internal_buffer);
assert(r5 ==0);
source_directory_members--;
}
} else if (de.d_type == DT_DIR) {
CroBuffer<2048,false> child_path(path, '/', de.d_name);
if (do_dir(src_root, dest_root, child_path)) {
source_directory_members--;
}
}
}
closedir(d);
if (!source_directory_members) {
CroBuffer<2048,false> src_dir_path(src_root,'/',path);
auto r5 = rmdir(src_dir_path.internal_buffer);
assert(r5 == 0);
return true;
} else {
return false;
}
}
#if 0
int main() {
crc_init(global_crc_table, false);
do_dir("/coding/misc/build/test1",
"/coding/misc/build/test2",
"");
}
#else
int main(int argc, char*argv[]) {
assert(argc == 3);
char p1[PATH_MAX];
char p2[PATH_MAX];
auto r1 = realpath(argv[1], p1);
assert(r1);
auto r2 = realpath(argv[2], p2);
assert(r2);
assert(strcmp(p1, p2)!=0);
do_dir(argv[1], argv[2], "");
}
#endif
Raw
set_cksums.cpp
// cro_build:noasan,release,raw, -static-libstdc++
// cro_build:run,raw, -static-libstdc++
#include <fcntl.h>
#include <sys/stat.h>
#include <unistd.h>
#include <sys/types.h>
#include <dirent.h>
#include <errno.h>
#include <signal.h>
#include <attr/xattr.h>
#include "cro_lib/cro_buffer.h"
typedef StringWithLength S;
#define _LOGNAME "set_cksums.cpp"
unsigned global_crc_table[256];
void crc_init(unsigned int *crc_table, int little_endian)
{
// Init the CRC32 table (big endian)
for (unsigned int i=0; i<256; i++) {
unsigned int j, c = little_endian ? i : i<<24;
for (j=8; j; j--)
if (little_endian) c = (c&1) ? (c>>1)^0xEDB88320 : c>>1;
else c=c&0x80000000 ? (c<<1)^0x04c11db7 : (c<<1);
crc_table[i] = c;
}
}
#if 0
// le
static unsigned cksum_le(unsigned crc, unsigned char c)
{
return global_crc_table[(crc^c)&0xff] ^ (crc>>8);
}
#endif
// be
static unsigned cksum_be(unsigned crc, unsigned char c)
{
return (crc<<8)^global_crc_table[(crc>>24)^c];
}
u64 progress = 0;
bool calculate_crc(int fd, u32&crc) {
u64 len = 0;
crc = (0) ? 0xffffffff : 0;
static u8 buffer[1<<20];
for (;;) {
int buffer_used = read(fd, buffer, sizeof buffer);
assert(buffer_used>=0);
for (int i=0;i<buffer_used;i++) {
crc = cksum_be(crc, buffer[i]);
}
progress+=buffer_used;
if (progress>100000000) {
progress-=100000000;
putchar('.');
fflush(stdout);
}
len+=buffer_used;
if (buffer_used != sizeof buffer) break;
}
if (1) {
while (len) {
crc = cksum_be(crc, len);
len >>= 8;
}
}
if (1) crc = ~crc;
return false;
}
char* current_checksum = NULL;
void print_status(int) {
if(current_checksum) {
cro_log("Current checksum is on:", current_checksum);
} else {
cro_log("No Current checksum");
}
}
void do_dir(S path) {
DIR*d = opendir(path.base);
assert(d);
for(;;) {
dirent de;
{
dirent*der;
auto r1 = readdir_r(d, &de, &der);
assert(r1==0);
if (!der) break;
assert(der == &de);
}
if (strcmp(de.d_name, ".")==0) {
continue;
} else if (strcmp(de.d_name, "..")==0){
continue;
}
CroBuffer<2048,false> child_path(path, '/', de.d_name);
if (de.d_type == DT_REG) {
#if 1
u32 crc;
auto r1 = getxattr(child_path.internal_buffer, "user.cro_cksum", &crc, sizeof crc);
if (r1 != sizeof crc) {
auto f = open(child_path.internal_buffer, O_RDONLY);
assert(f);
current_checksum = child_path.internal_buffer;
calculate_crc(f, crc);
current_checksum = NULL;
close(f);
auto r2 = setxattr(child_path.internal_buffer, "user.cro_cksum", &crc, sizeof crc, 0);
assert(r2 == 0);
//cro_log("did crc:", crc);
}
#else
u32 crc;
auto r1 = getxattr(child_path.internal_buffer, "user.asdf", &crc, sizeof crc);
if (r1 != sizeof crc) {
cro_log("Failed to get attribute for", child_path, "size:", r1, strerror(errno));
} else {
cro_log("Got size ", crc);
}
#endif
} else if (de.d_type == DT_DIR) {
do_dir(child_path);
}
}
closedir(d);
}
#if 0
int main() {
signal(SIGUSR1, print_status);
crc_init(global_crc_table, false);
do_dir("/coding/misc/build/test2");
//do_dir("/usr/games");
}
#else
int main(int argc, char*argv[]) {
signal(SIGUSR1, print_status);
crc_init(global_crc_table, false);
assert(argc == 2);
do_dir(argv[1]);
}
#endif
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment