Upload large files to zenodo and serve it as a zarr store (not working yet because zenodo doesn't support range header)

upload_zenodo.py

import requests
import os

ACCESS_TOKEN = '...'

r = requests.get('https://sandbox.zenodo.org/api/deposit/depositions', params={'access_token': ACCESS_TOKEN})
assert r.status_code == 200

headers = {"Content-Type": "application/json"}
params = {'access_token': ACCESS_TOKEN}

r = requests.post('https://sandbox.zenodo.org/api/deposit/depositions', 
                    params=params,
                   json={},
                   # Headers are not necessary here since "requests" automatically
                   # adds "Content-Type: application/json", because we're using
                   # the "json=" keyword argument
                   # headers=headers,
                   headers=headers)
assert r.status_code == 201
result = r.json()
bucket_url = result["links"]["bucket"]
deposition_id = result['id']

# Upload with the new API
FILE_PATH = './image.ome.tif_offsets.json'
_, filename = os.path.split(FILE_PATH)
with open(FILE_PATH, "rb") as fp:
    r = requests.put(
        "%s/%s" % (bucket_url, filename),
        data=fp,
        params=params,
    )

FILE_PATH = './3rd-acquisition-plate-8-2/image.ome.tif'
_, filename = os.path.split(FILE_PATH)
with open(FILE_PATH, "rb") as fp:
    r = requests.put(
        "%s/%s" % (bucket_url, filename),
        data=fp,
        params=params,
    )

print('File uploaded as ' + str(deposition_id))

VizarrReferenceStore-Zenodo.imjoy.html

<docs>
[TODO: write documentation for this plugin.]
</docs>

<config lang="json">
{
  "name": "VizarrReferenceStore",
  "type": "web-worker",
  "tags": [],
  "ui": "",
  "version": "0.1.0",
  "cover": "",
  "description": "Reading Vizarr using reference store",
  "icon": "extension",
  "inputs": null,
  "outputs": null,
  "api_version": "0.1.8",
  "env": "",
  "permissions": [],
  "requirements": ["https://unpkg.com/zarr@0.4.0/zarr.umd.js"],
  "dependencies": []
}
</config>

<script lang="javascript">

// Using the FileReferenceStore made by Trevor Manz
// https://observablehq.com/d/a417a770860fbb64
class FileReferenceStore {
  constructor(ref, target) {
    this.ref = ref;
    this.target = target;
  }

  _url(urlPath) {
    if(!urlPath){
      return this.target
    }
    const [protocol, path] = urlPath.split('://');
    if (protocol === 'https' || protocol === 'http') {
      return urlPath;
    }
    if (protocol === 'gc') {
      return 'https://storage.googleapis.com/' + path;
    }
    throw Error("Protocol not supported, got: " + JSON.stringify(protocol));
  }

  async getItem(key) {
    const res = this.ref[key];
    if (!res) {
      // Key not in store
      throw new zarr.KeyError(key);
    }
   
    if (res?.length !== 3) {
      // JSON data entry in reference 
      const meta = typeof res === 'string' ? res : JSON.stringify(res)
      const enc = new TextEncoder().encode(meta);
      return enc.buffer;
    }

    const [urlPath, offset, size] = res;
    const url = this._url(urlPath);
    const headers = {
      Range: `bytes=${offset}-${offset + size - 1}`
    };
    const value = await fetch(url, { headers, mode: 'cors'}).then(res => res.arrayBuffer());
    return value;
  }

  containsItem(key) {
    return key in this.ref;
  }

  static async fromUrl(url, targetUrl) {
    const ref = await fetch(url).then(res => res.json());
    return new FileReferenceStore(ref, targetUrl);
  }
}

const ZENODO = 'sandbox.zenodo.org'

// convert zenodo deposit url to enable CORS
async function convertZenodoFileUrl(url) {
  const myRegexp = new RegExp("https?:\/\/"+ZENODO+"\/record\/([a-zA-Z0-9-]+)\/files\/(.*)");
  const match = myRegexp.exec(url);
  if (!match || match.length !== 3) {
    throw new Error("Invalid zenodo url");
  }
  const [fullUrl, depositId, fileName] = match;
  const blob = await fetch(
    `https://${ZENODO}/api/records/${depositId}`
  ).then(r => r.blob());
  const data = JSON.parse(await new Response(blob).text());
  const fn = fileName.split("?")[0];
  const fileObj = data.files.filter(file => {
    return file.key === fn;
  })[0];
  return fileObj.links.self;
}

class ImJoyPlugin {
  async setup() {
    api.log('initialized')
  }

  async run(ctx) {
    const refUrl = await convertZenodoFileUrl('https://sandbox.zenodo.org/record/755844/files/image.ome.tif_offsets.json')
    const targetUrl = await convertZenodoFileUrl('https://sandbox.zenodo.org/record/755844/files/image.ome.tif')
    const store = await FileReferenceStore.fromUrl(refUrl, targetUrl);
    const viewer = await api.createWindow({src: 'https://hms-dbmi.github.io/vizarr'})
    const img = {
      "source": store,
      "name": "Image",
      "channel_axis": 1,
      "colors": ["#0000FF", "#FF0000", "#00FFFF", "#00FF00"],   // optional
      "names": ["DAPI", "red", "cyan", "green"],        // optional, default channel_X
      "visibilities": [true, true, true, true],           // optional, default True
      "contrast_limits": [[60, 500], [70, 300], [70, 5000], [70, 1500]], // optional, but recommended
    }
    await viewer.add_image(img)
  }
}

api.export(new ImJoyPlugin())
</script>