Last active
March 26, 2024 08:59
-
-
Save jay0lee/ccc780cc4fe32a942300 to your computer and use it in GitHub Desktop.
Upload 5tb Drive File
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
refresh_access_token () | |
{ | |
echo "Old access token: $access_token" | |
refresh_result=$(curl -s https://accounts.google.com/o/oauth2/token \ | |
-H "Content-Type: application/x-www-form-urlencoded" \ | |
-d refresh_token=$refresh_token \ | |
-d client_id=$client_id \ | |
-d client_secret=$client_secret \ | |
-d grant_type=refresh_token) | |
access_token=$(echo -e "$refresh_result" | \ | |
grep -Po '"access_token" *: *.*?[^\\]",' | \ | |
awk -F'"' '{ print $4 }') | |
expires_in=3600 | |
time_now=`date +%s` | |
expires_at=$(($time_now + $expires_in)) | |
echo -e "access_token=$access_token | |
refresh_token=$refresh_token | |
expires_at=$expires_at" > $my_creds | |
echo "New access token: $access_token" | |
echo "Expires at: $expires_at" | |
echo "Time now: $time_now" | |
} | |
# Store our credentials in our home directory with a file called .<script name> | |
my_creds=~/.`basename $0` | |
client_id='726549120613-5tteml7hdqvk8grb3afra32i3iddjnjb.apps.googleusercontent.com' | |
client_secret='Yuyb_xMSFo7MoWvNyqb44K4_' # not really a secret | |
if [ -s $my_creds ]; then | |
# if we already have a token stored, use it | |
. $my_creds | |
time_now=`date +%s` | |
else | |
scope='https://www.googleapis.com/auth/drive.file' | |
# Form the request URL | |
# http://goo.gl/U0uKEb | |
auth_url="https://accounts.google.com/o/oauth2/auth? | |
client_id=$client_id& | |
scope=$scope& | |
response_type=code& | |
redirect_uri=urn:ietf:wg:oauth:2.0:oob" | |
echo "Please go to:" | |
echo | |
echo "$auth_url" | |
echo | |
echo "after accepting, enter the code you are given:" | |
read auth_code | |
# swap authorization code for access and refresh tokens | |
# http://goo.gl/Mu9E5J | |
auth_result=$(curl -s https://accounts.google.com/o/oauth2/token \ | |
-H "Content-Type: application/x-www-form-urlencoded" \ | |
-d code=$auth_code \ | |
-d client_id=$client_id \ | |
-d client_secret=$client_secret \ | |
-d redirect_uri=urn:ietf:wg:oauth:2.0:oob \ | |
-d grant_type=authorization_code) | |
access_token=$(echo -e "$auth_result" | \ | |
grep -Po '"access_token" *: *.*?[^\\]",' | \ | |
awk -F'"' '{ print $4 }') | |
refresh_token=$(echo -e "$auth_result" | \ | |
grep -Po '"refresh_token" *: *.*?[^\\]",*' | \ | |
awk -F'"' '{ print $4 }') | |
expires_in=$(echo -e "$auth_result" | \ | |
grep -Po '"expires_in" *: *.*' | \ | |
awk -F' ' '{ print $3 }' | awk -F',' '{ print $1}') | |
time_now=`date +%s` | |
expires_at=$((time_now + expires_in - 60)) | |
echo -e "access_token=$access_token | |
refresh_token=$refresh_token | |
expires_at=$expires_at" > $my_creds | |
fi | |
# if our access token is expired, use the refresh token to get a new one | |
# http://goo.gl/71rN6V | |
if [ $time_now -gt $expires_at ]; then | |
refresh_access_token | |
fi | |
mbytes=20000 | |
chunk_size=`expr 1024 \* 1024 \* $mbytes` | |
if [ ! -f ~/zeros.gz ]; then | |
echo "Creating a gzipped 20gb file of zeros..." | |
dd if=/dev/zero bs=1M count=$mbytes | gzip --best > ~/zeros.gz | |
echo "done" | |
fi | |
file_size=`expr 5 \* 1024 \* 1024 \* 1000 \* 1000` | |
echo "Uploading a 5tb file in 20gb chunks..." | |
start_bytes=0 | |
end_bytes=`expr $chunk_size - 1` | |
location_url="https://www.googleapis.com/upload/drive/v3/files?\ | |
uploadType=resumable" | |
echo $location_url | |
filelocation=$(curl -s -i -X POST $location_url \ | |
-d '{"name": "test-5tb-file", "mimeType": "application/octet-stream"}' \ | |
-H "Content-Type: application/json" \ | |
-H "X-Upload-Content-Length: $file_size" \ | |
-H "X-Upload-Content-Type: applicaton/octet-stream" \ | |
-H "Authorization: Bearer $access_token" | \ | |
grep -Fi "location: ") | |
filelocation="${filelocation:10:-1}" | |
n=1 | |
token_refreshes=0 | |
while : | |
do | |
echo "Uploading bytes ${start_bytes} - ${end_bytes}/${file_size}..." | |
starttime=$(date +%s) | |
put_result=$(curl -s -o /dev/null -w "%{http_code}" \ | |
--upload ~/zeros.gz \ | |
-H "Content-Type: application/octet-stream" \ | |
-H "Authorization: Bearer $access_token" \ | |
-H "Content-Range: bytes $start_bytes-$end_bytes/$file_size" \ | |
-H "Content-Encoding: gzip" \ | |
"${filelocation}") | |
echo $put_result | |
if [ $put_result \> 499 ]; | |
then | |
echo "Got $put_result. Sleeping $n..." | |
sleep $n | |
n=`expr $n \* 2` | |
if [ $n \> 60 ]; | |
then | |
n=60 | |
fi | |
continue | |
fi | |
if [ $put_result \> 399 ]; | |
then | |
echo "Got $put_result. Refreshing token..." | |
refresh_access_token | |
token_refreshes=`expr $token_refreshes + 1` | |
if [ $token_refreshes \> 3 ]; | |
then | |
echo "Refreshing token doesn't seem to fix. Giving up." | |
break | |
fi | |
continue | |
fi | |
if [ $put_result \> 299 ]; | |
then | |
echo "Got $put_result. Continuing..." | |
endtime=$(date +%s) | |
timediff=$(( $endtime - $starttime )) | |
echo "chunk took $timediff seconds..." | |
n=1 | |
token_refreshes=0 | |
else | |
echo "Got $put_result. We are done." | |
break | |
fi | |
start_bytes=`expr $end_bytes + 1` | |
end_bytes=`expr $end_bytes + $chunk_size` | |
if (( $end_bytes > $file_size )); then | |
$end_bytes=$filee_size | |
fi | |
echo $(echo "${end_bytes} / ${file_size}" | bc -l) | |
done |
Does it work? I dont have an unlimited plan.
And how many of them can you upload?
how do i upload it?
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Attempt to create a 5tb Google Drive file full of zeroes. We use zeroes because they compress really well :-)