Motoko + ZSH CLI Upload

main.mo

// This part of the example is a bit long-winded. The upload logic bits are at the very bottom.

shared ({ caller = creator }) actor class MyCanister() = canister {


    // State


    // The upload buffer, for adding additional assets.
    private let buffer : Buffer.Buffer<Blob> = Buffer.Buffer(0);

    // A denormalized hashmap for looking up assets.
    private let files : HashMap.HashMap<Text, Nat> = HashMap.HashMap(0, Text.equal, Text.hash);

    // We store assets in a buffer, where each asset has some metadata and an asset payload.
    // Assets are retrieved from the buffer by searching on their metadata.
    private let assets : Buffer.Buffer<Types.Record> = Buffer.Buffer(0);


    //  Internals


    // Writes a new asset to state, including the denormalized index
    private func _addAsset (record : Types.Record) : Result.Result<(), Text> {
        switch (files.get(record.meta.filename)) {
            case (?i) {
                // Add asset to state
                assets.put(i, record);
                #ok();
            };
            case _ {
                // Store a denormalized index of filename to asset index
                files.put(record.meta.filename, assets.size());
                // Add asset to state
                assets.add(record);
                #ok();
            };
        };
    };

    // Determine whether an asset has a given tag.
    private func _assetHasTag (
        asset   : Types.Record,
        tag     : Text,
    ) : Bool {
        for (t in asset.meta.tags.vals()) {
            if (t == tag) return true;
        };
        return false;
    };


    // Utilities


    // Turn a list of blobs into one blob.
    // NOTE: Probably a problem with my upload methodology that I need to use this in some places?
    // i.e. a payload should perhaps ultimately be stored as a blob, not a list of blobs?
    public func _flattenPayload (payload : [Blob]) : Blob {
        Blob.fromArray(
            Array.foldLeft<Blob, [Nat8]>(payload, [], func (a : [Nat8], b : Blob) {
                Array.append(a, Blob.toArray(b));
            })
        );
    };

    // Find the first asset with a given tag.
    public func _findTag (tag : Text) : ?Types.Record {
        Array.find<Types.Record>(assets.toArray(), func (asset : Types.Record) {
            _assetHasTag(asset, tag);
        });
    };

    // Find the first asset with all given tag.
    public func _findTags (tags : [Text]) : ?Types.Record {
        Array.find<Types.Record>(assets.toArray(), func (asset : Types.Record) {
            for (tag in tags.vals()) {
                if (_assetHasTag(asset, tag) == false) return false;
            };
            return true;
        });
    };

    // Find all asset with a given tag.
    public func _findAllTag (tag : Text) : [Types.Record] {
        Array.filter<Types.Record>(assets.toArray(), func (asset : Types.Record) {
            _assetHasTag(asset, tag);
        });
    };


    // Pub API


    // Retrieve an asset.
    public func getAssetByName (
        filename : Text,
    ) : ?Types.Record {
        switch (files.get(filename)) {
            case (?index) ?assets.get(index);
            case _ null;
        };
    };

    // Retrieve the asset manifest.
    public func getManifest () : [Types.Record] {
        assets.toArray();
    };


    // Admin API


    // Upload bytes into the buffer.
    // @auth: admin
    public func upload(
        caller  : Principal,
        bytes   : [Blob],
    ) : () {
        // NOTE: _isAdmin not implemented in this example
        assert(admins._isAdmin(caller));
        for (byte in bytes.vals()) {
            buffer.add(byte);
        }
    };

    // Finalize the upload buffer into an asset.
    // @auth: admin
    public func uploadFinalize(
        caller      : Principal,
        contentType : Text,
        meta        : Types.Meta,
    ) : Result.Result<(), Text> {
        // NOTE: _isAdmin not implemented in this example
        assert(admins._isAdmin(caller));
        switch (
            _addAsset({
                asset = {
                    contentType = contentType;
                    payload = buffer.toArray();
                };
                meta;
            })
        ) {
            case (#err(msg)) return #err(msg);
            case _ {
                buffer.clear();
                return #ok();
            }
        };
    };

    // Clear the upload buffer
    // @auth: admin
    public func uploadClear(
        caller : Principal,
    ) : () {
        // NOTE: _isAdmin not implemented in this example
        assert(admins._isAdmin(caller));
        buffer.clear();
    };

}

upload.zsh

#!/bin/zsh

file=${1}
filename=$(echo $file | sed -E "s/.+\///")
fileextension=$(echo $file | sed -E "s/.+\.//")
name=${2}
tags=("${(@s/ /)3}")
description=${4}
mime=${5:-image/$fileextension}
network=${6:-local}
threshold=${7:-100000}


echo "$network Emptying buffer..."
dfx canister --network $network call legends-staging uploadClear

i=0
byteSize=${#$(od -An -v -tuC $file)[@]}
echo "$network Uploading asset \"$filename\", size: $byteSize"
while [ $i -le $byteSize ]; do
    echo "chunk #$(($i/$threshold+1))..."
    dfx canister --network $network call legends-staging upload "( vec {\
        vec { $(for byte in ${(j:;:)$(od -An -v -tuC $file)[@]:$i:$threshold}; echo "$byte;") };\
    })"
    i=$(($i+$threshold))
done
echo "$network Finalizing asset \"$filename\""
dfx canister --network $network call legends-staging uploadFinalize "(\
    \"\",\
    record {\
        \"name\" = \"$name\";\
        \"filename\" = \"$filename\";\
        \"tags\" = vec { $(for tag in $tags; echo \"$tag\"\;) };\
        \"description\" = \"$description\";\
    }\
)"