Uploading GCS Objects in parallel using PHP SDK

testParallelUploadToGcs.php

<?php

require __DIR__ . "/vendor/autoload.php";

use Google\Cloud\Storage\StorageClient;
use GuzzleHttp\Promise;

# control the number of objects to upload
$NUM_OBJECTS = 5;
$storage = new StorageClient();
$bucketName = 'test_parallel_uploads'.time();
$bucket = $storage->createBucket($bucketName);

$cmd = 'mkdir -p folder_upload_test';
exec($cmd);
$cmd = 'seq '. $NUM_OBJECTS . ' | xargs -I {} bash -c "perl -le \'print \"xyz\" x 10485760\' > folder_upload_test/{}.txt"';
exec($cmd);
$folderPath = __DIR__ . '/folder_upload_test';

// Remove '.' and '..' from the file list
$files = scandir($folderPath);
$files = array_diff($files, ['.', '..']);

echo "####### START #############\n";
$ts = time();
echo "$ts,start,ALL\n";
// Upload each file
$promises = [];
foreach ($files as $file) {
    if (is_dir($file)) {
        echo "$file,directory,skip\n";
        continue;
    }
    $objectName = $folderPath.'/'.$file;
    $tstart = time();
    echo "$tstart,start,$file\n";
    $promises[] = $bucket->uploadAsync(
        fopen($folderPath . '/' . $file, 'r'),
        ['name' => $objectName]
    );
}
$responses = Promise\Utils::unwrap($promises);
// Wait for the requests to complete, even if some of them fail
// $responses = Promise\Utils::settle($promises)->wait();

$te = time();
$tdiff = $te-$ts;
echo "$te,end,ALL,$tdiff\n";

exec('rm -rf folder_upload_test');
foreach ($bucket->objects() as $object) {
    $object->delete();
}
$bucket->delete();
echo "######### END #############\n";

# nproc on my machine gives 48
# 1 object of 30 MB each takes ~3 seconds
# 10 object of 30 MB each takes ~4 seconds
# 100 object of 30 MB each takes ~10 seconds
# 500 object of 30 MB each takes ~20 seconds