Skip to content

Instantly share code, notes, and snippets.

@jay0lee
Last active March 26, 2024 08:59
Show Gist options
  • Save jay0lee/ccc780cc4fe32a942300 to your computer and use it in GitHub Desktop.
Save jay0lee/ccc780cc4fe32a942300 to your computer and use it in GitHub Desktop.
Upload 5tb Drive File
refresh_access_token ()
{
echo "Old access token: $access_token"
refresh_result=$(curl -s https://accounts.google.com/o/oauth2/token \
-H "Content-Type: application/x-www-form-urlencoded" \
-d refresh_token=$refresh_token \
-d client_id=$client_id \
-d client_secret=$client_secret \
-d grant_type=refresh_token)
access_token=$(echo -e "$refresh_result" | \
grep -Po '"access_token" *: *.*?[^\\]",' | \
awk -F'"' '{ print $4 }')
expires_in=3600
time_now=`date +%s`
expires_at=$(($time_now + $expires_in))
echo -e "access_token=$access_token
refresh_token=$refresh_token
expires_at=$expires_at" > $my_creds
echo "New access token: $access_token"
echo "Expires at: $expires_at"
echo "Time now: $time_now"
}
# Store our credentials in our home directory with a file called .<script name>
my_creds=~/.`basename $0`
client_id='726549120613-5tteml7hdqvk8grb3afra32i3iddjnjb.apps.googleusercontent.com'
client_secret='Yuyb_xMSFo7MoWvNyqb44K4_' # not really a secret
if [ -s $my_creds ]; then
# if we already have a token stored, use it
. $my_creds
time_now=`date +%s`
else
scope='https://www.googleapis.com/auth/drive.file'
# Form the request URL
# http://goo.gl/U0uKEb
auth_url="https://accounts.google.com/o/oauth2/auth?
client_id=$client_id&
scope=$scope&
response_type=code&
redirect_uri=urn:ietf:wg:oauth:2.0:oob"
echo "Please go to:"
echo
echo "$auth_url"
echo
echo "after accepting, enter the code you are given:"
read auth_code
# swap authorization code for access and refresh tokens
# http://goo.gl/Mu9E5J
auth_result=$(curl -s https://accounts.google.com/o/oauth2/token \
-H "Content-Type: application/x-www-form-urlencoded" \
-d code=$auth_code \
-d client_id=$client_id \
-d client_secret=$client_secret \
-d redirect_uri=urn:ietf:wg:oauth:2.0:oob \
-d grant_type=authorization_code)
access_token=$(echo -e "$auth_result" | \
grep -Po '"access_token" *: *.*?[^\\]",' | \
awk -F'"' '{ print $4 }')
refresh_token=$(echo -e "$auth_result" | \
grep -Po '"refresh_token" *: *.*?[^\\]",*' | \
awk -F'"' '{ print $4 }')
expires_in=$(echo -e "$auth_result" | \
grep -Po '"expires_in" *: *.*' | \
awk -F' ' '{ print $3 }' | awk -F',' '{ print $1}')
time_now=`date +%s`
expires_at=$((time_now + expires_in - 60))
echo -e "access_token=$access_token
refresh_token=$refresh_token
expires_at=$expires_at" > $my_creds
fi
# if our access token is expired, use the refresh token to get a new one
# http://goo.gl/71rN6V
if [ $time_now -gt $expires_at ]; then
refresh_access_token
fi
mbytes=20000
chunk_size=`expr 1024 \* 1024 \* $mbytes`
if [ ! -f ~/zeros.gz ]; then
echo "Creating a gzipped 20gb file of zeros..."
dd if=/dev/zero bs=1M count=$mbytes | gzip --best > ~/zeros.gz
echo "done"
fi
file_size=`expr 5 \* 1024 \* 1024 \* 1000 \* 1000`
echo "Uploading a 5tb file in 20gb chunks..."
start_bytes=0
end_bytes=`expr $chunk_size - 1`
location_url="https://www.googleapis.com/upload/drive/v3/files?\
uploadType=resumable"
echo $location_url
filelocation=$(curl -s -i -X POST $location_url \
-d '{"name": "test-5tb-file", "mimeType": "application/octet-stream"}' \
-H "Content-Type: application/json" \
-H "X-Upload-Content-Length: $file_size" \
-H "X-Upload-Content-Type: applicaton/octet-stream" \
-H "Authorization: Bearer $access_token" | \
grep -Fi "location: ")
filelocation="${filelocation:10:-1}"
n=1
token_refreshes=0
while :
do
echo "Uploading bytes ${start_bytes} - ${end_bytes}/${file_size}..."
starttime=$(date +%s)
put_result=$(curl -s -o /dev/null -w "%{http_code}" \
--upload ~/zeros.gz \
-H "Content-Type: application/octet-stream" \
-H "Authorization: Bearer $access_token" \
-H "Content-Range: bytes $start_bytes-$end_bytes/$file_size" \
-H "Content-Encoding: gzip" \
"${filelocation}")
echo $put_result
if [ $put_result \> 499 ];
then
echo "Got $put_result. Sleeping $n..."
sleep $n
n=`expr $n \* 2`
if [ $n \> 60 ];
then
n=60
fi
continue
fi
if [ $put_result \> 399 ];
then
echo "Got $put_result. Refreshing token..."
refresh_access_token
token_refreshes=`expr $token_refreshes + 1`
if [ $token_refreshes \> 3 ];
then
echo "Refreshing token doesn't seem to fix. Giving up."
break
fi
continue
fi
if [ $put_result \> 299 ];
then
echo "Got $put_result. Continuing..."
endtime=$(date +%s)
timediff=$(( $endtime - $starttime ))
echo "chunk took $timediff seconds..."
n=1
token_refreshes=0
else
echo "Got $put_result. We are done."
break
fi
start_bytes=`expr $end_bytes + 1`
end_bytes=`expr $end_bytes + $chunk_size`
if (( $end_bytes > $file_size )); then
$end_bytes=$filee_size
fi
echo $(echo "${end_bytes} / ${file_size}" | bc -l)
done
@jay0lee
Copy link
Author

jay0lee commented Sep 27, 2014

Attempt to create a 5tb Google Drive file full of zeroes. We use zeroes because they compress really well :-)

@HaraldWeber
Copy link

Does it work? I dont have an unlimited plan.
And how many of them can you upload?

@porg1
Copy link

porg1 commented Oct 21, 2020

how do i upload it?

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment