Skip to content

Instantly share code, notes, and snippets.

@killerbees19

killerbees19/README.md

Last active Aug 1, 2020
Embed
What would you like to do?
SIMPLE BIDIRECTIONAL SYNC WITH HARDLINKS
mkdir ~/.sync
chmod -c 0700 ~/.sync

editor ~/sync.php
chmod +x ~/sync.php

~/sync.php
# 2-Wege-Synchronisierung von <A> zu <B> und umgekehrt auf Basis von Hardlinks. Siehe ~/sync.php und ~/.sync!
# Für den lockfile Befehl muss das Paket procmail installiert sein. Das ist optional, aber sehr empfehlenswert.
# Dateien werden nicht sofort gelöscht! Sie landen in einem Backupordner und werden erst nach einer Woche gelöscht.
*/12 * * * * which lockfile 1>/dev/null && lockfile -r 5 -l 86400 ~/.sync/.lock || exit 1; ionice -c 3 ~/sync.php; rm -f ~/.sync/.lock
@weekly find ~/.sync/*/.bkp -mtime "+7" -type "f" -print -delete
#!/usr/bin/php
<?php
########################################################################
# #
# SIMPLE BIDIRECTIONAL SYNC WITH HARDLINKS #
# ---------------------------------------- #
# v1.0.1 (2020-07-01) by killerbees19 #
# #
# Synchronisiert Ordner und Dateien von <A> zu <b> und umgekehrt. #
# Dabei werden nur Ordner neu angelegt, Dateien werden hart verlinkt. #
# #
# Symbolische Verknüpfungen werden nicht unterstützt und ignoriert. #
# Es ist geplant, dass diese Unterstützung später nachgerüstet wird. #
# Dabei sollen sogar absolute Pfade in Symlinks umgeschrieben werden. #
# #
# Die Berechtigungen (chmod) werden bei Ordnern korrekt übernommen. #
# Der Besitzer und die Gruppe (chown) werden nicht extra angepasst! #
# Dateien, die nicht dem gleichen Benutzer gehören, erzeugen Fehler. #
# #
# Änderungen werden in beiden Richtungen erkannt. Für diesen Zweck #
# gibt es den Ordner ~/.sync. Nach dem Ändern einer Regel in $rules #
# sollte das dazu gehörende Verzeichnis in ~/.sync gelöscht werden! #
# Andernfalls kann es zu seltsamen Aktionen und Fehlern kommen. #
# #
# WICHTIG: Dieses Script beachtet keine .gitignore Dateien! #
# Man sollte bei der Konfiguration somit sehr aufmerksam sein. #
# #
# HINWEIS: Wenn Ordnerberechtigungen an beiden Orten gleichzeitig #
# geändert werden, gewinnen immer die Berechtigungen von Ordner B! #
# #
# NOTIZ: Leere Ordner werden erst beim zweiten Durchlauf gelöscht... #
# Beim ersten Durchlauf werden nur die enthaltenen Dateien entfernt. #
# #
# https://gist.github.com/cf266f21950b1dc1c3156714897b6a09 #
# #
########################################################################
set_error_handler('exceptionErrorHandler');
error_reporting(-1); setlocale(LC_ALL, 'C');
# TODO: Lockfile?
# ...
# TODO: Support for symlinks?
# ...
# TODO: Update mtime of directories?
# ...
if(!isset($_SERVER['HOME']) || ($_ = realpath($_SERVER['HOME'])) === false)
{
$_ = __dir__;
}
define('SYNC_INTERNAL_DIR', sprintf('%s/.sync', $_));
define('SYNC_A_DIR', sprintf('%s/dir-A', $_));
define('SYNC_B_DIR', sprintf('%s/dir-B', $_));
define('SYNC_DRY_RUN', true);
define('SYNC_VERBOSE', true);
define('SYNC_DEBUG', false);
unset($_);
if(!file_exists(SYNC_INTERNAL_DIR) || !is_dir(SYNC_INTERNAL_DIR))
{
throw new RuntimeException(sprintf('Internal sync directory not found: %s', SYNC_INTERNAL_DIR));
}
// order=true (include vor exclude)
// order=false (exclude vor include)
$rules =
[
'test' => ['path' => 'sync-test', 'recursive' => true, 'order' => true, 'include' => ['*/', '*/*'], 'exclude' => ['*.*']],
];
$uids = [];
foreach($rules as $id => $rule)
{
if(!($id = basename($id)) || isset($uids[$id]))
{
throw new RuntimeException(sprintf('Rule ID is invalid or not unique: %s', $id));
}
else
{
$uids[$id] = true;
}
debug(str_repeat('-', 80));
debug(sprintf('-- Current rule: %s', $id));
debug(str_repeat('-', 80));
$a = realpath(sprintf('%s/%s', SYNC_A_DIR, $rule['path']));
$b = realpath(sprintf('%s/%s', SYNC_B_DIR, $rule['path']));
if(!$a || !$b)
{
throw new RuntimeException(sprintf('[%s] Path not found: a=%s; b=%s', $id, $a, $b));
}
$sync = sprintf('%s/%s', SYNC_INTERNAL_DIR, $id);
$async = sprintf('%s/a', $sync);
$bsync = sprintf('%s/b', $sync);
if(!file_exists($sync))
{
if(!mkdir($sync, 0700))
{
throw new Exception();
}
}
else if(is_link($sync) || !is_dir($sync))
{
throw new Exception();
}
if(!file_exists($async))
{
if(!mkdir($async, 0700))
{
throw new Exception();
}
}
else if(is_link($async) || !is_dir($async))
{
throw new Exception();
}
if(!file_exists($bsync))
{
if(!mkdir($bsync, 0700))
{
throw new Exception();
}
}
else if(is_link($bsync) || !is_dir($bsync))
{
throw new Exception();
}
$bkp = sprintf('%s/.bkp', $sync);
if(!file_exists($bkp))
{
if(!mkdir($bkp, 0700))
{
throw new Exception();
}
}
else if(is_link($bkp) || !is_dir($bkp))
{
throw new Exception();
}
if(!($async = realpath($async)) || !($bsync = realpath($bsync)) || !($bkp = realpath($bkp)))
{
throw new Exception();
}
$alist = getFileList($a, $rule);
$blist = getFileList($b, $rule);
$asynclist = getFileList($async, $rule);
$bsynclist = getFileList($bsync, $rule);
debug(str_repeat('-', 80));
debug(sprintf('-- [%s] Sync from A to B', $id));
debug(str_repeat('-', 80));
foreach($alist as $file => $data)
{
$arealpath = sprintf('%s/%s', $a, $file);
$asyncpath = sprintf('%s/%s', $async, $file);
$brealpath = sprintf('%s/%s', $b, $file);
$bsyncpath = sprintf('%s/%s', $bsync, $file);
if(!isset($blist[$file]))
{
if(isset($asynclist[$file]) && isset($bsynclist[$file]))
{
if(deleteFile($arealpath, $bkp))
{
unset($alist[$file]);
deleteFile($asyncpath, null, true);
unset($asynclist[$file]);
deleteFile($bsyncpath, null, true);
unset($bsynclist[$file]);
}
}
else
{
if($data['type'] === 'd')
{
if(!isset($asynclist[$file]))
{
createDirectory($asyncpath, $data['perms'], true);
}
createDirectory($brealpath, $data['perms']);
if(!isset($bsynclist[$file]))
{
createDirectory($bsyncpath, $data['perms'], true);
}
}
else
{
if(!isset($asynclist[$file]))
{
createHardLink($arealpath, $asyncpath, true);
}
createHardLink($arealpath, $brealpath);
if(!isset($bsynclist[$file]))
{
createHardLink($brealpath, $bsyncpath, true);
}
}
# Update internal array?
# ...
}
}
else if($blist[$file]['type'] !== $data['type'])
{
throw new RuntimeException(sprintf('[%s] Unsupported type change: %s', $id, $file));
}
else if($data['type'] === 'd' && $asynclist[$file]['perms'] !== $data['perms'] && $blist[$file]['perms'] !== $data['perms'])
{
chmodFile($asyncpath, $data['perms'], true);
$asynclist[$file]['perms'] = $data['perms'];
chmodFile($bsyncpath, $data['perms'], true);
$bsynclist[$file]['perms'] = $data['perms'];
chmodFile($brealpath, $data['perms']);
$blist[$file]['perms'] = $data['perms'];
}
}
unset($file, $data);
debug(str_repeat('-', 80));
debug(sprintf('-- [%s] Sync from B to A', $id));
debug(str_repeat('-', 80));
foreach($blist as $file => $data)
{
$brealpath = sprintf('%s/%s', $b, $file);
$bsyncpath = sprintf('%s/%s', $bsync, $file);
$arealpath = sprintf('%s/%s', $a, $file);
$asyncpath = sprintf('%s/%s', $async, $file);
if(!isset($alist[$file]))
{
if(isset($bsynclist[$file]) && isset($asynclist[$file]))
{
if(deleteFile($brealpath, $bkp))
{
unset($blist[$file]);
deleteFile($bsyncpath, null, true);
unset($bsynclist[$file]);
deleteFile($asyncpath, null, true);
unset($asynclist[$file]);
}
}
else
{
if($data['type'] === 'd')
{
if(!isset($bsynclist[$file]))
{
createDirectory($bsyncpath, $data['perms'], true);
}
createDirectory($arealpath, $data['perms']);
if(!isset($asynclist[$file]))
{
createDirectory($asyncpath, $data['perms'], true);
}
}
else
{
if(!isset($bsynclist[$file]))
{
createHardLink($brealpath, $bsyncpath, true);
}
createHardLink($brealpath, $arealpath);
if(!isset($asynclist[$file]))
{
createHardLink($arealpath, $asyncpath, true);
}
}
# Update internal array?
# ...
}
}
else if($alist[$file]['type'] !== $data['type'])
{
throw new RuntimeException(sprintf('[%s] Unsupported type change: %s', $id, $file));
}
else if($data['type'] === 'd' && $bsynclist[$file]['perms'] !== $data['perms'] && $alist[$file]['perms'] !== $data['perms'])
{
chmodFile($bsyncpath, $data['perms'], true);
$bsynclist[$file]['perms'] = $data['perms'];
chmodFile($asyncpath, $data['perms'], true);
$asynclist[$file]['perms'] = $data['perms'];
chmodFile($arealpath, $data['perms']);
$alist[$file]['perms'] = $data['perms'];
}
}
unset($file, $data);
debug(str_repeat('-', 80));
debug(sprintf('-- [%s] Internal cleanup of A', $id));
debug(str_repeat('-', 80));
foreach($asynclist as $file => $data)
{
$syncpath = sprintf('%s/%s', $async, $file);
if(!isset($alist[$file]) && !isset($blist[$file]))
{
if(deleteFile($syncpath, null, true))
{
unset($asynclist[$file]);
}
}
}
debug(str_repeat('-', 80));
debug(sprintf('-- [%s] Internal cleanup of B', $id));
debug(str_repeat('-', 80));
foreach($bsynclist as $file => $data)
{
$syncpath = sprintf('%s/%s', $bsync, $file);
if(!isset($blist[$file]) && !isset($alist[$file]))
{
if(deleteFile($syncpath, null, true))
{
unset($bsynclist[$file]);
}
}
}
}
function deleteFile($filename, $bkp = null, $internal = false)
{
if(is_dir($filename))
{
return deleteDirectory($filename, $bkp, $internal);
}
if($bkp === null)
{
debug(sprintf('rm: %s', $filename), true, $internal);
if(!SYNC_DRY_RUN && !unlink($filename))
{
throw new RuntimeException('Could not delete file');
}
}
else
{
$bkpfile = sprintf('%s/%d_%d_%s', $bkp, round(microtime(true) * 1000), filemtime($filename), base32::encode($filename));
debug(sprintf('mv: %s -> %s', $filename, $bkpfile), true, $internal);
if(!SYNC_DRY_RUN && !rename($filename, $bkpfile))
{
throw new RuntimeException('Could not move file');
}
try
{
touch($bkpfile);
}
catch(Exception $e)
{
/* ... */
}
}
return !SYNC_DRY_RUN;
}
function deleteDirectory($dirname, $bkp = null, $internal = false)
{
if(!(new FilesystemIterator($dirname))->valid())
{
# $bkp beachten?
# ...
debug(sprintf('rmdir: %s', $dirname), true, $internal);
if(!SYNC_DRY_RUN && !rmdir($dirname))
{
throw new RuntimeException('Could not delete directory');
}
return !SYNC_DRY_RUN;
}
debug(sprintf('Ignoring non-empty directory: %s', $dirname));
return false;
}
function chmodFile($filename, $mode, $internal = false)
{
debug(sprintf('chmod: %o %s', $mode, $filename), true, $internal);
if(!SYNC_DRY_RUN && !chmod($filename, $mode))
{
throw new RuntimeException('Could not chmod file');
}
return !SYNC_DRY_RUN;
}
function createHardLink($target, $link, $internal = false)
{
debug(sprintf('ln: %s -> %s', $target, $link), true, $internal);
if(!SYNC_DRY_RUN && !link($target, $link))
{
throw new RuntimeException('Could not create hardlink');
}
return !SYNC_DRY_RUN;
}
function createDirectory($pathname, $mode, $internal = false)
{
debug(sprintf('mkdir: %s', $pathname), true, $internal);
if(!SYNC_DRY_RUN && !mkdir($pathname))
{
throw new RuntimeException('Could not create directory');
}
return chmodFile($pathname, $mode, $internal);
}
function debug($msg, $action = false, $internal = false)
{
if(SYNC_DEBUG || ($action && !$internal && (SYNC_DRY_RUN || SYNC_VERBOSE)))
{
fwrite(STDERR, ($action && SYNC_DRY_RUN ? '[DRY] ' : '') . $msg . "\n");
}
}
function prepareRuleMatch($_)
{
return sprintf('#^%s$#', str_replace('\\*', '.+', preg_quote($_, '#')));
}
function getFileList($start, $rule)
{
$return = [];
$scan = ['.'];
$imatch = array_map('prepareRuleMatch', $rule['include']);
$ematch = array_map('prepareRuleMatch', $rule['exclude']);
while($dir = array_shift($scan))
{
$realdir = sprintf('%s/%s', $start, $dir);
if($dh = opendir($realdir))
{
while(($file = readdir($dh)) !== false)
{
if($file !== '..' && $file !== '.')
{
$fullpath = sprintf('%s/%s', $realdir, $file);
$path = substr(sprintf('%s/%s', $dir, $file), 2);
if(is_link($fullpath))
{
#throw new RuntimeException(sprintf('Symlinks are not supported: %s', $fullpath));
debug(sprintf('Ignoring unsupported symlink: %s', $fullpath));
continue;
}
else if(is_dir($fullpath))
{
// Fake name for regex check...
$bcheck = sprintf('%s/', $file);
$acheck = sprintf('%s/', $path);
}
else
{
$bcheck = $file;
$acheck = $path;
}
$do = null;
if($rule['order'])
{
foreach($imatch as $i)
{
if(strpos($i, '/') === false)
{
$_ = $bcheck;
}
else
{
$_ = $acheck;
}
if(preg_match($i, $_))
{
$do = true;
break;
}
}
unset($i);
if($do === null)
{
foreach($ematch as $e)
{
if(strpos($e, '/') === false)
{
$_ = $bcheck;
}
else
{
$_ = $acheck;
}
if(preg_match($e, $_))
{
$do = false;
break;
}
}
unset($e);
}
}
else
{
foreach($ematch as $e)
{
if(strpos($e, '/') === false)
{
$_ = $bcheck;
}
else
{
$_ = $acheck;
}
if(preg_match($e, $_))
{
$do = false;
break;
}
}
unset($e);
if($do === null)
{
foreach($imatch as $i)
{
if(strpos($i, '/') === false)
{
$_ = $bcheck;
}
else
{
$_ = $acheck;
}
if(preg_match($i, $_))
{
$do = true;
break;
}
}
unset($i);
}
}
if($do === false)
{
debug(sprintf('(-) %s', $fullpath));
continue;
}
else if($do === true)
{
debug(sprintf('(+) %s', $fullpath));
}
else
{
debug(sprintf('(?) %s', $fullpath));
}
if(is_dir($fullpath))
{
if($rule['recursive'])
{
$scan[] = sprintf('%s/%s', $dir, $file);
$return[$path] = ['type' => 'd', 'perms' => fileperms($fullpath)];
}
}
else if(is_file($fullpath))
{
$return[$path] = ['type' => 'f', 'perms' => fileperms($fullpath)];
}
else
{
throw new RuntimeException(sprintf('Unsupported type: %s', $fullpath));
}
}
}
closedir($dh);
}
else
{
throw new Exception();
}
}
return $return;
}
function exceptionErrorHandler($severity, $message, $file, $line)
{
if (!(error_reporting() & $severity))
{
return;
}
throw new ErrorException($message, 0, $severity, $file, $line);
}
// Base32 Encode/Decode by Bryan Ruiz <bryan@bryanruiz.com>
// http://php.net/manual/de/function.base-convert.php#102232
abstract class Base32
{
private static $map =
[
'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', // 7
'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', // 15
'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', // 23
'Y', 'Z', '2', '3', '4', '5', '6', '7', // 31
'=' // padding char
];
private static $flippedMap =
[
'A' => '0', 'B' => '1', 'C' => '2', 'D' => '3', 'E' => '4', 'F' => '5', 'G' => '6', 'H' => '7',
'I' => '8', 'J' => '9', 'K' => '10', 'L' => '11', 'M' => '12', 'N' => '13', 'O' => '14', 'P' => '15',
'Q' => '16', 'R' => '17', 'S' => '18', 'T' => '19', 'U' => '20', 'V' => '21', 'W' => '22', 'X' => '23',
'Y' => '24', 'Z' => '25', '2' => '26', '3' => '27', '4' => '28', '5' => '29', '6' => '30', '7' => '31',
];
// Use padding false when encoding for urls
public static function encode($input, $padding = true)
{
if(empty($input))
{
return '';
}
$base32 = '';
$binaryString = '';
$input = str_split($input);
$ci = count($input);
for($i = 0; $i < $ci; $i++)
{
$binaryString .= str_pad(base_convert(ord($input[$i]), 10, 2), 8, '0', STR_PAD_LEFT);
}
$fiveBitBinaryArray = str_split($binaryString, 5);
$i = 0;
$ci = count($fiveBitBinaryArray);
while($i < $ci)
{
$base32 .= self::$map[base_convert(str_pad($fiveBitBinaryArray[$i], 5, '0'), 2, 10)];
$i++;
}
if($padding && ($x = strlen($binaryString) % 40) != 0)
{
switch($x)
{
case 8: $base32 .= str_repeat(self::$map[32], 6); break;
case 16: $base32 .= str_repeat(self::$map[32], 4); break;
case 24: $base32 .= str_repeat(self::$map[32], 3); break;
case 32: $base32 .= self::$map[32]; break;
}
}
return $base32;
}
public static function decode($input)
{
if(empty($input))
{
return;
}
$paddingCharCount = substr_count($input, self::$map[32]);
$allowedValues = [6, 4, 3, 1, 0];
if(!in_array($paddingCharCount, $allowedValues))
{
return false;
}
for($i = 0; $i < 4; $i++)
{
if($paddingCharCount == $allowedValues[$i] && substr($input, -($allowedValues[$i])) != str_repeat(self::$map[32], $allowedValues[$i]))
{
return false;
}
}
$input = str_replace('=', '', $input);
$input = str_split($input);
$binaryString = '';
$ci = count($input);
for($i = 0; $i < $ci; $i += 8)
{
$x = '';
if(!in_array($input[$i], self::$map))
{
return false;
}
for($j = 0; $j < 8; $j++)
{
$x .= str_pad(base_convert(@self::$flippedMap[@$input[$i + $j]], 10, 2), 5, '0', STR_PAD_LEFT);
}
$eightBits = str_split($x, 8);
$cz = count($eightBits);
for($z = 0; $z < $cz; $z++)
{
$binaryString .= ( ($y = chr(base_convert($eightBits[$z], 2, 10))) || ord($y) == 48 ) ? $y : '';
}
}
return $binaryString;
}
}
?>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment