Instantly share code, notes, and snippets.
larsvilhuber/Uploading to repositories
Created Mar 13, 2019
Uploading to repositories
#!/bin/bash | |
# | |
# Upload large file to a Figshare item. | |
# Copied mostly from Figshare bash script example. | |
# | |
# exit script if any command fails | |
set -e | |
if [ -f .config ]; then | |
source .config | |
fi | |
if [ -z "$FIGSHARE_ACCESS_TOKEN" ]; then | |
echo 'FIGSHARE_ACCESS_TOKEN required. Please set it via env variable or .config.' | |
exit 2 | |
fi | |
ACCESS_TOKEN=$FIGSHARE_ACCESS_TOKEN | |
FILE_PATH="$1" | |
ITEM_ID=$2 | |
if [ -z "$FILE_PATH" ]; then | |
echo "Usage: $0 <file path> [<item id>]" | |
echo '' | |
echo 'A new item will be created if item id is not provided.' | |
echo '' | |
RESPONSE=$(curl -s -H 'Authorization: token '$ACCESS_TOKEN -X GET "https://api.figshare.com/v2/account/articles") | |
echo "The item list dict contains: "$RESPONSE | |
echo '' | |
exit 2 | |
fi | |
# exit 0 | |
FILE_NAME=$(basename "$FILE_PATH") | |
# #################################################################################### | |
#Retrieve the file size and MD5 values for the item which needs to be uploaded | |
FILE_SIZE=$(stat -c%s $FILE_PATH) | |
echo "Calculating MD5" | |
MD5=($(pv $FILE_PATH | md5sum)) | |
echo "Uploading $FILE_PATH ($FILE_SIZE bytes, md5: $MD5) to item id $ITEM_ID" | |
if [ -z "$ITEM_ID" ]; then | |
# Create a new item | |
echo 'Creating a new item...' | |
TITLE="$FILE_NAME" | |
ITEM_JSON="{\"title\": \"$TITLE\"}" | |
RESPONSE=$(curl -s -d "$ITEM_JSON" -H 'Authorization: token '$ACCESS_TOKEN -H 'Content-Type: application/json' -X POST "https://api.figshare.com/v2/account/articles") | |
echo "The location of the created item is "$RESPONSE | |
echo '' | |
# Retrieve item id | |
echo 'Retrieving the item id...' | |
ITEM_ID=$(echo "$RESPONSE" | sed -r "s/.*\/([0-9]+).*/\1/") | |
echo "The item id is "$ITEM_ID | |
echo '' | |
fi | |
# List item files | |
echo 'Retrieving the item files...' | |
FILES_LIST=$(curl -s -H 'Authorization: token '$ACCESS_TOKEN -X GET "https://api.figshare.com/v2/account/articles/$ITEM_ID/files") | |
echo 'The files list of the newly-create item should be an empty one. Returned results: '$FILES_LIST | |
echo '' | |
# Initiate new upload: | |
echo 'A new upload had been initiated...' | |
RESPONSE=$(curl -s -d '{"md5": "'${MD5}'", "name": "'${FILE_NAME}'", "size": '${FILE_SIZE}'}' -H 'Content-Type: application/json' -H 'Authorization: token '$ACCESS_TOKEN -X POST "https://api.figshare.com/v2/account/articles/$ITEM_ID/files") | |
echo $RESPONSE | |
echo '' | |
# Retrieve file id | |
echo 'The file id is retrieved...' | |
FILE_ID=$(echo "$RESPONSE" | sed -r "s/.*\/([0-9]+).*/\1/") | |
echo 'The file id is: '$FILE_ID | |
echo '' | |
# Retrieve the upload url | |
echo 'Retrieving the upload URL...' | |
RESPONSE=$(curl -s -H 'Authorization: token '$ACCESS_TOKEN -X GET "https://api.figshare.com/v2/account/articles/$ITEM_ID/files/$FILE_ID") | |
UPLOAD_URL=$(echo "$RESPONSE" | sed -r 's/.*"upload_url":\s"([^"]+)".*/\1/') | |
echo 'The upload URL is: '$UPLOAD_URL | |
echo '' | |
# Retrieve the upload parts | |
echo 'Retrieving the part value...' | |
RESPONSE=$(curl -s -H 'Authorization: token '$ACCESS_TOKEN -X GET "$UPLOAD_URL") | |
PARTS_SIZE=$(echo "$RESPONSE" | sed -r 's/"endOffset":([0-9]+).*/\1/' | sed -r 's/.*,([0-9]+)/\1/') | |
PARTS_SIZE=$(($PARTS_SIZE+1)) | |
echo 'The part value is: '$PARTS_SIZE | |
echo '' | |
# Split item into needed parts | |
echo 'Spliting the provided item into parts process had begun...' | |
split -b$PARTS_SIZE $FILE_PATH part_ --numeric=1 --suffix-length=3 | |
echo 'Process completed!' | |
# Retrive the number of parts | |
MAX_PART=$((($FILE_SIZE+$PARTS_SIZE-1)/$PARTS_SIZE)) | |
echo 'The number of parts is: '$MAX_PART | |
echo '' | |
# Perform the PUT operation of parts | |
echo 'Perform the PUT operation of parts...' | |
for ((i=1; i<=$MAX_PART; i++)) | |
do | |
PART_VALUE='part_'$(printf "%03d" $i) | |
RESPONSE=$(curl -s -H 'Authorization: token '$ACCESS_TOKEN -X PUT "$UPLOAD_URL/$i" --data-binary @$PART_VALUE) | |
echo "Done uploading part nr: $i/"$MAX_PART | |
done | |
echo 'Process was finished!' | |
echo '' | |
# Complete upload | |
echo 'Completing the file upload...' | |
RESPONSE=$(curl -s -H 'Authorization: token '$ACCESS_TOKEN -X POST "https://api.figshare.com/v2/account/articles/$ITEM_ID/files/$FILE_ID") | |
echo 'Done!' | |
echo '' | |
#remove the part files | |
rm part_* | |
# List all of the existing items | |
RESPONSE=$(curl -s -H 'Authorization: token '$ACCESS_TOKEN -X GET "https://api.figshare.com/v2/account/articles") | |
echo 'New list of items: '$RESPONSE | |
echo '' | |
figshare: https://github.com/elifesciences/datacapsule-crossref | |
zenodo: https://developers.zenodo.org/#quickstart-upload | |
dataverse: https://github.com/IQSS/dataverse-uploader/wiki/DVUploader,-a-Command-line-Bulk-Uploader-for-Dataverse |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment