Skip to content

Instantly share code, notes, and snippets.

@EthraZa
Last active June 6, 2019 12:00
Show Gist options
  • Save EthraZa/d044a7d40e6f38daa8e7e9370d92be3f to your computer and use it in GitHub Desktop.
Save EthraZa/d044a7d40e6f38daa8e7e9370d92be3f to your computer and use it in GitHub Desktop.
Spider Struct :: Mirror files from some remote place, based on a source directory structure, to a distination directory
<?php
/**
* Spider Struct - by Allan.Brazute
* Mirror files from some remote place, based on a source directory structure, to a distination directory
*
* Usage: spiderstruct.php /local_src_structure_path/ /output_path/ http://base_url/
*/
$localSrc = (isset($argv[1]) && $argv[1])? $argv[1] : '';
$localDst = (isset($argv[2]) && $argv[2])? $argv[2] : '';
$remoteSrc = (isset($argv[3]) && $argv[3])? $argv[3] : '';
$spin = array('-','\\','/');
$spincc = 0;
$filescc = 0;
if (!$localSrc || !$localDst || !$remoteSrc) {
echo "\n" . 'Usage: spiderstruct.php /local_src_structure_path/ /output_path/ http://base_url/' . "\n\n";
die();
}
$files = rsearch($localSrc);
$filesct = count($files);
foreach($files as $file) {
$fl = strtr($file, array($localSrc => ""));
$url = $remoteSrc . $fl;
$dst = $localDst . $fl;
$dir = dirname($dst);
$filescc++;
if (is_file($file)) {
if (!file_exists($dst)) {
if ($spincc > 2) $spincc = 0;
echo "\033[2K" . $spin[$spincc++] . ' Grabbing (' . $filescc .'/' . $filesct . ') ' . $url . " \r";
$content = file_get_contents($url);
if ($content !== false) {
if (!file_exists($dir)) {
mkdir($dir, 0777, true);
}
file_put_contents($dst, $content);
} else {
echo '- Failed (' . $filescc .'/' . $filesct . ') ' . $url . " \n";
}
} else {
echo '- Skipping (' . $filescc .'/' . $filesct . ') ' . $url . " \n";
}
}
}
echo '- Done with ' . $filescc . " files \n";
function rsearch($folder, $pattern = '/.*/') {
$dir = new RecursiveDirectoryIterator($folder);
$ite = new RecursiveIteratorIterator($dir);
$files = new RegexIterator($ite, $pattern, RegexIterator::GET_MATCH);
$fileList = array();
foreach($files as $file) {
$fileList = array_merge($fileList, $file);
}
return $fileList;
}
?>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment